From: Mike Blumenkrantz Date: Tue, 21 Mar 2023 14:18:22 +0000 (-0400) Subject: zink: move all barrier-related functions to c++ X-Git-Tag: upstream/23.3.3~10957 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=bf0af0f8edeed8e2e8c74c2a4c3f4baae75ead24;p=platform%2Fupstream%2Fmesa.git zink: move all barrier-related functions to c++ this will enable a lot of deduplication Part-of: --- diff --git a/src/gallium/drivers/zink/meson.build b/src/gallium/drivers/zink/meson.build index b976b30..98070ee 100644 --- a/src/gallium/drivers/zink/meson.build +++ b/src/gallium/drivers/zink/meson.build @@ -42,6 +42,7 @@ files_libzink = files( 'zink_screen.c', 'zink_state.c', 'zink_surface.c', + 'zink_synchronization.cpp', ) zink_device_info = custom_target( diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index c0410fa..ab15ed0 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -3546,714 +3546,6 @@ zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t memcpy(ctx->sample_locations, locations, size); } -static VkAccessFlags -access_src_flags(VkImageLayout layout) -{ - switch (layout) { - case VK_IMAGE_LAYOUT_UNDEFINED: - return VK_ACCESS_NONE; - - case VK_IMAGE_LAYOUT_GENERAL: - return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; - - case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: - case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT: - return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: - return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT; - - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: - case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: - return VK_ACCESS_SHADER_READ_BIT; - - case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: - return VK_ACCESS_TRANSFER_READ_BIT; - - case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: - return VK_ACCESS_TRANSFER_WRITE_BIT; - - case VK_IMAGE_LAYOUT_PREINITIALIZED: - return VK_ACCESS_HOST_WRITE_BIT; - - case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: - return VK_ACCESS_NONE; - - default: - unreachable("unexpected layout"); - } -} - -static VkAccessFlags -access_dst_flags(VkImageLayout layout) -{ - switch (layout) { - case VK_IMAGE_LAYOUT_UNDEFINED: - return VK_ACCESS_NONE; - - case VK_IMAGE_LAYOUT_GENERAL: - return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; - - case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: - case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT: - return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: - return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - - case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: - return VK_ACCESS_SHADER_READ_BIT; - - case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: - return VK_ACCESS_TRANSFER_READ_BIT; - - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: - return VK_ACCESS_SHADER_READ_BIT; - - case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: - return VK_ACCESS_TRANSFER_WRITE_BIT; - - case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: - return VK_ACCESS_NONE; - - default: - unreachable("unexpected layout"); - } -} - -static VkPipelineStageFlags -pipeline_dst_stage(VkImageLayout layout) -{ - switch (layout) { - case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: - return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: - return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; - - case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: - return VK_PIPELINE_STAGE_TRANSFER_BIT; - case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: - return VK_PIPELINE_STAGE_TRANSFER_BIT; - - case VK_IMAGE_LAYOUT_GENERAL: - return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; - - case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: - case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: - return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - - default: - return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; - } -} - -#define ALL_READ_ACCESS_FLAGS \ - (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \ - VK_ACCESS_INDEX_READ_BIT | \ - VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \ - VK_ACCESS_UNIFORM_READ_BIT | \ - VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \ - VK_ACCESS_SHADER_READ_BIT | \ - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \ - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \ - VK_ACCESS_TRANSFER_READ_BIT |\ - VK_ACCESS_HOST_READ_BIT |\ - VK_ACCESS_MEMORY_READ_BIT |\ - VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\ - VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\ - VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\ - VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ - VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\ - VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\ - VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\ - VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ - VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR) - - -bool -zink_resource_access_is_write(VkAccessFlags flags) -{ - return (flags & ~ALL_READ_ACCESS_FLAGS) > 0; -} - -bool -zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - if (!pipeline) - pipeline = pipeline_dst_stage(new_layout); - if (!flags) - flags = access_dst_flags(new_layout); - return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline || - (res->obj->access & flags) != flags || - zink_resource_access_is_write(res->obj->access) || - zink_resource_access_is_write(flags); -} - -bool -zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - if (!pipeline) - pipeline = pipeline_dst_stage(new_layout); - if (!flags) - flags = access_dst_flags(new_layout); - - VkImageSubresourceRange isr = { - res->aspect, - 0, VK_REMAINING_MIP_LEVELS, - 0, VK_REMAINING_ARRAY_LAYERS - }; - *imb = (VkImageMemoryBarrier){ - VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - NULL, - res->obj->access ? res->obj->access : access_src_flags(res->layout), - flags, - res->layout, - new_layout, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - res->obj->image, - isr - }; - return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline); -} - -static bool -zink_resource_image_barrier2_init(VkImageMemoryBarrier2 *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - if (!pipeline) - pipeline = pipeline_dst_stage(new_layout); - if (!flags) - flags = access_dst_flags(new_layout); - - VkImageSubresourceRange isr = { - res->aspect, - 0, VK_REMAINING_MIP_LEVELS, - 0, VK_REMAINING_ARRAY_LAYERS - }; - *imb = (VkImageMemoryBarrier2){ - VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, - NULL, - res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - res->obj->access ? res->obj->access : access_src_flags(res->layout), - pipeline, - flags, - res->layout, - new_layout, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - res->obj->image, - isr - }; - return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline); -} - -static inline bool -is_shader_pipline_stage(VkPipelineStageFlags pipeline) -{ - return pipeline & GFX_SHADER_BITS; -} - -static void -resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline) -{ - assert(res->obj->is_buffer); - if (res->bind_count[0] - res->so_bind_count > 0) { - if ((res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) || - (util_bitcount(res->vbo_bind_mask) != res->bind_count[0] && !is_shader_pipline_stage(pipeline))) - /* gfx rebind */ - _mesa_set_add(ctx->need_barriers[0], res); - } - if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)) - /* compute rebind */ - _mesa_set_add(ctx->need_barriers[1], res); -} - -static inline bool -unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write) -{ - /* if all usage is unordered, keep unordered */ - if (res->obj->unordered_read && res->obj->unordered_write) - return true; - /* if testing write access but have any ordered read access, cannot promote */ - if (is_write && zink_batch_usage_matches(res->obj->bo->reads, ctx->batch.state) && !res->obj->unordered_read) - return false; - /* if write access is unordered or nonexistent, always promote */ - return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes, ctx->batch.state); -} - -VkCommandBuffer -zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst) -{ - bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0; - if (src) - unordered_exec &= unordered_res_exec(ctx, src, false); - if (dst) - unordered_exec &= unordered_res_exec(ctx, dst, true); - if (src) - src->obj->unordered_read = unordered_exec; - if (dst) - dst->obj->unordered_write = unordered_exec; - if (!unordered_exec || ctx->unordered_blitting) - zink_batch_no_rp(ctx); - if (unordered_exec) { - ctx->batch.state->has_barriers = true; - return ctx->batch.state->barrier_cmdbuf; - } - return ctx->batch.state->cmdbuf; -} - -static void -resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline) -{ - assert(!res->obj->is_buffer); - assert(!ctx->blitting); - - bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - /* if this is a non-shader barrier and there are binds, always queue a shader barrier */ - bool is_shader = is_shader_pipline_stage(pipeline); - if ((is_shader || !res->bind_count[is_compute]) && - /* if no layout change is needed between gfx and compute, do nothing */ - !res->bind_count[!is_compute] && (!is_compute || !res->fb_bind_count)) - return; - - if (res->bind_count[!is_compute] && is_shader) { - /* if the layout is the same between gfx and compute, do nothing */ - if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute)) - return; - } - /* queue a layout change if a layout change will be needed */ - if (res->bind_count[!is_compute]) - _mesa_set_add(ctx->need_barriers[!is_compute], res); - /* also queue a layout change if this is a non-shader layout */ - if (res->bind_count[is_compute] && !is_shader) - _mesa_set_add(ctx->need_barriers[is_compute], res); -} - -void -zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, - VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - VkImageMemoryBarrier imb; - if (!pipeline) - pipeline = pipeline_dst_stage(new_layout); - - if (!zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline)) - return; - bool is_write = zink_resource_access_is_write((VkAccessFlags)imb.dstAccessMask); - VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - assert(new_layout); - enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; - if (!res->obj->access_stage || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw)) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - } - bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); - VKCTX(CmdPipelineBarrier)( - cmdbuf, - res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - pipeline, - 0, - 0, NULL, - 0, NULL, - 1, &imb - ); - zink_cmd_debug_marker_end(ctx, cmdbuf, marker); - - resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); - - if (is_write) - res->obj->last_write = (VkAccessFlags)imb.dstAccessMask; - - res->obj->access = (VkAccessFlags)imb.dstAccessMask; - res->obj->access_stage = pipeline; - res->layout = new_layout; - if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) - zink_resource_copies_reset(res); -} - -void -zink_resource_image_barrier2(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - VkImageMemoryBarrier2 imb; - if (!pipeline) - pipeline = pipeline_dst_stage(new_layout); - - if (!zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline)) - return; - bool is_write = zink_resource_access_is_write((VkAccessFlags)imb.dstAccessMask); - VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - assert(new_layout); - enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; - if (!res->obj->access_stage || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw)) - imb.srcAccessMask = 0; - if (res->obj->needs_zs_evaluate) - imb.pNext = &res->obj->zs_evaluate; - res->obj->needs_zs_evaluate = false; - if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { - imb.srcQueueFamilyIndex = res->queue; - imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; - res->queue = VK_QUEUE_FAMILY_IGNORED; - } - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 0, - NULL, - 0, - NULL, - 1, - &imb - }; - bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); - VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); - zink_cmd_debug_marker_end(ctx, cmdbuf, marker); - - resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); - - if (is_write) - res->obj->last_write = (VkAccessFlags)imb.dstAccessMask; - - res->obj->access = (VkAccessFlags)imb.dstAccessMask; - res->obj->access_stage = pipeline; - res->layout = new_layout; - if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) - zink_resource_copies_reset(res); -} - -bool -zink_check_unordered_transfer_access(struct zink_resource *res, unsigned level, const struct pipe_box *box) -{ - /* always barrier against previous non-transfer writes */ - bool non_transfer_write = res->obj->last_write && res->obj->last_write != VK_ACCESS_TRANSFER_WRITE_BIT; - /* must barrier if clobbering a previous write */ - bool transfer_clobber = res->obj->last_write == VK_ACCESS_TRANSFER_WRITE_BIT && zink_resource_copy_box_intersects(res, level, box); - return non_transfer_write || transfer_clobber; -} - -bool -zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) -{ - return res->obj->access && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !unordered_res_exec(ctx, res, false); -} - -void -zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box) -{ - if (res->obj->copies_need_reset) - zink_resource_copies_reset(res); - /* skip TRANSFER_DST barrier if no intersection from previous copies */ - if (res->layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL || - zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || - zink_check_unordered_transfer_access(res, level, box)) { - zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - } else { - res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; - res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT; - res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; - } - zink_resource_copy_box_add(res, level, box); -} - -bool -zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) -{ - if (res->obj->copies_need_reset) - zink_resource_copies_reset(res); - bool unordered = true; - struct pipe_box box = {(int)offset, 0, 0, (int)size, 0, 0}; - bool can_unordered_write = unordered_res_exec(ctx, res, true); - /* must barrier if something read the valid buffer range */ - bool valid_read = (res->obj->access || res->obj->unordered_access) && - util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write; - if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { - zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - unordered = res->obj->unordered_write; - } else { - res->obj->unordered_access = VK_ACCESS_TRANSFER_WRITE_BIT; - res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT; - res->obj->unordered_access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; - - ctx->batch.state->unordered_write_access |= VK_ACCESS_TRANSFER_WRITE_BIT; - ctx->batch.state->unordered_write_stages |= VK_PIPELINE_STAGE_TRANSFER_BIT; - if (!zink_resource_usage_matches(res, ctx->batch.state)) { - res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; - res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; - res->obj->ordered_access_is_copied = true; - } - } - zink_resource_copy_box_add(res, 0, &box); - /* this return value implies that the caller could do an unordered op on this resource */ - return unordered; -} - -VkPipelineStageFlags -zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage) -{ - switch (stage) { - case VK_SHADER_STAGE_VERTEX_BIT: - return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; - case VK_SHADER_STAGE_FRAGMENT_BIT: - return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; - case VK_SHADER_STAGE_GEOMETRY_BIT: - return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT; - case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT: - return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT; - case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT: - return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT; - case VK_SHADER_STAGE_COMPUTE_BIT: - return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - default: - unreachable("unknown shader stage bit"); - } -} - -ALWAYS_INLINE static VkPipelineStageFlags -pipeline_access_stage(VkAccessFlags flags) -{ - if (flags & (VK_ACCESS_UNIFORM_READ_BIT | - VK_ACCESS_SHADER_READ_BIT | - VK_ACCESS_SHADER_WRITE_BIT)) - return VK_PIPELINE_STAGE_TASK_SHADER_BIT_NV | - VK_PIPELINE_STAGE_MESH_SHADER_BIT_NV | - VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR | - VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | - VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | - VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | - VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - return VK_PIPELINE_STAGE_TRANSFER_BIT; -} - -ALWAYS_INLINE static bool -buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline, bool unordered) -{ - return zink_resource_access_is_write(unordered ? res->obj->unordered_access : res->obj->access) || - zink_resource_access_is_write(flags) || - ((unordered ? res->obj->unordered_access_stage : res->obj->access_stage) & pipeline) != pipeline || - ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags; -} - -void -zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - if (!pipeline) - pipeline = pipeline_access_stage(flags); - - bool is_write = zink_resource_access_is_write(flags); - bool unordered = unordered_res_exec(ctx, res, is_write); - if (!buffer_needs_barrier(res, flags, pipeline, unordered)) - return; - enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; - bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); - bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); - bool unordered_usage_matches = res->obj->unordered_access && usage_matches; - if (completed) { - /* reset access on complete */ - res->obj->access = VK_ACCESS_NONE; - res->obj->access_stage = VK_PIPELINE_STAGE_NONE; - res->obj->last_write = VK_ACCESS_NONE; - } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) { - /* always reset propagated access to avoid weirdness */ - res->obj->access = VK_ACCESS_NONE; - res->obj->access_stage = VK_PIPELINE_STAGE_NONE; - } else if (!unordered && !unordered_usage_matches) { - /* reset unordered access on first ordered barrier */ - res->obj->unordered_access = VK_ACCESS_NONE; - res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; - } - if (!usage_matches) { - /* reset unordered on first new cmdbuf barrier */ - res->obj->unordered_access = VK_ACCESS_NONE; - res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; - res->obj->ordered_access_is_copied = false; - } - /* unordered barriers can be skipped if either: - * - there is no current-batch unordered access - * - the unordered access is not write access - */ - bool can_skip_unordered = !unordered ? false : (!unordered_usage_matches || !zink_resource_access_is_write(res->obj->unordered_access)); - /* ordered barriers can be skipped if both: - * - there is no current access - * - there is no current-batch unordered access - */ - bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches); - - if (!can_skip_unordered && !can_skip_ordered) { - VkMemoryBarrier bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; - bmb.pNext = NULL; - VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; - if (unordered) { - stages = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcAccessMask = res->obj->access; - } - - VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - bool marker = false; - if (unlikely(zink_tracing)) { - char buf[4096]; - bool first = true; - unsigned idx = 0; - u_foreach_bit64(bit, flags) { - if (!first) - buf[idx++] = '|'; - idx += snprintf(&buf[idx], sizeof(buf) - idx, "%s", vk_AccessFlagBits_to_str((VkAccessFlagBits)(1ul<obj->last_write = flags; - if (unordered) { - /* these should get automatically emitted during submission */ - res->obj->unordered_access = flags; - res->obj->unordered_access_stage = pipeline; - if (is_write) { - ctx->batch.state->unordered_write_access |= flags; - ctx->batch.state->unordered_write_stages |= pipeline; - } - } - if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) { - res->obj->access = flags; - res->obj->access_stage = pipeline; - res->obj->ordered_access_is_copied = unordered; - } - if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write) - zink_resource_copies_reset(res); -} - -void -zink_resource_buffer_barrier2(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) -{ - if (!pipeline) - pipeline = pipeline_access_stage(flags); - - bool is_write = zink_resource_access_is_write(flags); - bool unordered = unordered_res_exec(ctx, res, is_write); - if (!buffer_needs_barrier(res, flags, pipeline, unordered)) - return; - enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; - bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); - bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); - bool unordered_usage_matches = res->obj->unordered_access && usage_matches; - if (completed) { - /* reset access on complete */ - res->obj->access = VK_ACCESS_NONE; - res->obj->access_stage = VK_PIPELINE_STAGE_NONE; - res->obj->last_write = VK_ACCESS_NONE; - } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) { - /* always reset propagated access to avoid weirdness */ - res->obj->access = VK_ACCESS_NONE; - res->obj->access_stage = VK_PIPELINE_STAGE_NONE; - } else if (!unordered && !unordered_usage_matches) { - /* reset unordered access on first ordered barrier */ - res->obj->unordered_access = VK_ACCESS_NONE; - res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; - } - if (!usage_matches) { - /* reset unordered on first new cmdbuf barrier */ - res->obj->unordered_access = VK_ACCESS_NONE; - res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; - res->obj->ordered_access_is_copied = false; - } - /* unordered barriers can be skipped if either: - * - there is no current-batch unordered access - * - the unordered access is not write access - */ - bool can_skip_unordered = !unordered ? false : (!unordered_usage_matches || !zink_resource_access_is_write(res->obj->unordered_access)); - /* ordered barriers can be skipped if both: - * - there is no current access - * - there is no current-batch unordered access - */ - bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches); - - if (!can_skip_unordered && !can_skip_ordered) { - VkMemoryBarrier2 bmb; - bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; - bmb.pNext = NULL; - VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; - if (unordered) { - bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; - bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; - } else { - bmb.srcStageMask = stages; - bmb.srcAccessMask = res->obj->access; - } - bmb.dstStageMask = pipeline; - bmb.dstAccessMask = flags; - VkDependencyInfo dep = { - VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - NULL, - 0, - 1, - &bmb, - 0, - NULL, - 0, - NULL - }; - VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); - bool marker = false; - if (unlikely(zink_tracing)) { - char buf[4096]; - bool first = true; - unsigned idx = 0; - u_foreach_bit64(bit, flags) { - if (!first) - buf[idx++] = '|'; - idx += snprintf(&buf[idx], sizeof(buf) - idx, "%s", vk_AccessFlagBits_to_str((VkAccessFlagBits)(1ul<obj->last_write = flags; - if (unordered) { - /* these should get automatically emitted during submission */ - res->obj->unordered_access = flags; - res->obj->unordered_access_stage = pipeline; - if (is_write) { - ctx->batch.state->unordered_write_access |= flags; - ctx->batch.state->unordered_write_stages |= pipeline; - } - } - if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) { - res->obj->access = flags; - res->obj->access_stage = pipeline; - res->obj->ordered_access_is_copied = unordered; - } - if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write) - zink_resource_copies_reset(res); -} - static void zink_flush(struct pipe_context *pctx, struct pipe_fence_handle **pfence, diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index d29296f..e109bec 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -193,7 +193,9 @@ bool zink_cmd_debug_marker_begin(struct zink_context *ctx, VkCommandBuffer cmdbuf, const char *fmt, ...); void zink_cmd_debug_marker_end(struct zink_context *ctx, VkCommandBuffer cmdbuf,bool emitted); - +void +zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src, + unsigned dst_offset, unsigned src_offset, unsigned size); #ifdef __cplusplus } #endif @@ -260,9 +262,6 @@ bool zink_use_dummy_attachments(const struct zink_context *ctx); void zink_set_color_write_enables(struct zink_context *ctx); -void -zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src, - unsigned dst_offset, unsigned src_offset, unsigned size); void zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src, diff --git a/src/gallium/drivers/zink/zink_synchronization.cpp b/src/gallium/drivers/zink/zink_synchronization.cpp new file mode 100644 index 0000000..feccbb9 --- /dev/null +++ b/src/gallium/drivers/zink/zink_synchronization.cpp @@ -0,0 +1,740 @@ +/* + * Copyright © 2023 Valve Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Mike Blumenkrantz + */ + +#include "zink_batch.h" +#include "zink_context.h" +#include "zink_descriptors.h" +#include "zink_resource.h" +#include "zink_screen.h" + + +static VkAccessFlags +access_src_flags(VkImageLayout layout) +{ + switch (layout) { + case VK_IMAGE_LAYOUT_UNDEFINED: + return VK_ACCESS_NONE; + + case VK_IMAGE_LAYOUT_GENERAL: + return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT: + return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT; + + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + return VK_ACCESS_SHADER_READ_BIT; + + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + return VK_ACCESS_TRANSFER_READ_BIT; + + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + return VK_ACCESS_TRANSFER_WRITE_BIT; + + case VK_IMAGE_LAYOUT_PREINITIALIZED: + return VK_ACCESS_HOST_WRITE_BIT; + + case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: + return VK_ACCESS_NONE; + + default: + unreachable("unexpected layout"); + } +} + +static VkAccessFlags +access_dst_flags(VkImageLayout layout) +{ + switch (layout) { + case VK_IMAGE_LAYOUT_UNDEFINED: + return VK_ACCESS_NONE; + + case VK_IMAGE_LAYOUT_GENERAL: + return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT: + return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + return VK_ACCESS_SHADER_READ_BIT; + + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + return VK_ACCESS_TRANSFER_READ_BIT; + + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: + return VK_ACCESS_SHADER_READ_BIT; + + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + return VK_ACCESS_TRANSFER_WRITE_BIT; + + case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: + return VK_ACCESS_NONE; + + default: + unreachable("unexpected layout"); + } +} + +static VkPipelineStageFlags +pipeline_dst_stage(VkImageLayout layout) +{ + switch (layout) { + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; + + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + return VK_PIPELINE_STAGE_TRANSFER_BIT; + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + return VK_PIPELINE_STAGE_TRANSFER_BIT; + + case VK_IMAGE_LAYOUT_GENERAL: + return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: + return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + + default: + return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + } +} + +#define ALL_READ_ACCESS_FLAGS \ + (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \ + VK_ACCESS_INDEX_READ_BIT | \ + VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \ + VK_ACCESS_UNIFORM_READ_BIT | \ + VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \ + VK_ACCESS_SHADER_READ_BIT | \ + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \ + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \ + VK_ACCESS_TRANSFER_READ_BIT |\ + VK_ACCESS_HOST_READ_BIT |\ + VK_ACCESS_MEMORY_READ_BIT |\ + VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\ + VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\ + VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\ + VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ + VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\ + VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\ + VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\ + VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ + VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR) + + +bool +zink_resource_access_is_write(VkAccessFlags flags) +{ + return (flags & ~ALL_READ_ACCESS_FLAGS) > 0; +} + +bool +zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + if (!flags) + flags = access_dst_flags(new_layout); + return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline || + (res->obj->access & flags) != flags || + zink_resource_access_is_write(res->obj->access) || + zink_resource_access_is_write(flags); +} + +bool +zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + if (!flags) + flags = access_dst_flags(new_layout); + + VkImageSubresourceRange isr = { + res->aspect, + 0, VK_REMAINING_MIP_LEVELS, + 0, VK_REMAINING_ARRAY_LAYERS + }; + *imb = VkImageMemoryBarrier { + VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + NULL, + res->obj->access ? res->obj->access : access_src_flags(res->layout), + flags, + res->layout, + new_layout, + VK_QUEUE_FAMILY_IGNORED, + VK_QUEUE_FAMILY_IGNORED, + res->obj->image, + isr + }; + return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline); +} + +static bool +zink_resource_image_barrier2_init(VkImageMemoryBarrier2 *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + if (!flags) + flags = access_dst_flags(new_layout); + + VkImageSubresourceRange isr = { + res->aspect, + 0, VK_REMAINING_MIP_LEVELS, + 0, VK_REMAINING_ARRAY_LAYERS + }; + *imb = VkImageMemoryBarrier2 { + VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + NULL, + res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + res->obj->access ? res->obj->access : access_src_flags(res->layout), + pipeline, + flags, + res->layout, + new_layout, + VK_QUEUE_FAMILY_IGNORED, + VK_QUEUE_FAMILY_IGNORED, + res->obj->image, + isr + }; + return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline); +} + +static inline bool +is_shader_pipline_stage(VkPipelineStageFlags pipeline) +{ + return pipeline & GFX_SHADER_BITS; +} + +static void +resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline) +{ + assert(res->obj->is_buffer); + if (res->bind_count[0] - res->so_bind_count > 0) { + if ((res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) || + (util_bitcount(res->vbo_bind_mask) != res->bind_count[0] && !is_shader_pipline_stage(pipeline))) + /* gfx rebind */ + _mesa_set_add(ctx->need_barriers[0], res); + } + if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)) + /* compute rebind */ + _mesa_set_add(ctx->need_barriers[1], res); +} + +static inline bool +unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write) +{ + /* if all usage is unordered, keep unordered */ + if (res->obj->unordered_read && res->obj->unordered_write) + return true; + /* if testing write access but have any ordered read access, cannot promote */ + if (is_write && zink_batch_usage_matches(res->obj->bo->reads, ctx->batch.state) && !res->obj->unordered_read) + return false; + /* if write access is unordered or nonexistent, always promote */ + return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes, ctx->batch.state); +} + +VkCommandBuffer +zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst) +{ + bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0; + if (src) + unordered_exec &= unordered_res_exec(ctx, src, false); + if (dst) + unordered_exec &= unordered_res_exec(ctx, dst, true); + if (src) + src->obj->unordered_read = unordered_exec; + if (dst) + dst->obj->unordered_write = unordered_exec; + if (!unordered_exec || ctx->unordered_blitting) + zink_batch_no_rp(ctx); + if (unordered_exec) { + ctx->batch.state->has_barriers = true; + return ctx->batch.state->barrier_cmdbuf; + } + return ctx->batch.state->cmdbuf; +} + +static void +resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline) +{ + assert(!res->obj->is_buffer); + assert(!ctx->blitting); + + bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + /* if this is a non-shader barrier and there are binds, always queue a shader barrier */ + bool is_shader = is_shader_pipline_stage(pipeline); + if ((is_shader || !res->bind_count[is_compute]) && + /* if no layout change is needed between gfx and compute, do nothing */ + !res->bind_count[!is_compute] && (!is_compute || !res->fb_bind_count)) + return; + + if (res->bind_count[!is_compute] && is_shader) { + /* if the layout is the same between gfx and compute, do nothing */ + if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute)) + return; + } + /* queue a layout change if a layout change will be needed */ + if (res->bind_count[!is_compute]) + _mesa_set_add(ctx->need_barriers[!is_compute], res); + /* also queue a layout change if this is a non-shader layout */ + if (res->bind_count[is_compute] && !is_shader) + _mesa_set_add(ctx->need_barriers[is_compute], res); +} + +void +zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, + VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + VkImageMemoryBarrier imb; + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + + if (!zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline)) + return; + bool is_write = zink_resource_access_is_write((VkAccessFlags)imb.dstAccessMask); + VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + assert(new_layout); + enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; + if (!res->obj->access_stage || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw)) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + } + bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); + VKCTX(CmdPipelineBarrier)( + cmdbuf, + res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + pipeline, + 0, + 0, NULL, + 0, NULL, + 1, &imb + ); + zink_cmd_debug_marker_end(ctx, cmdbuf, marker); + + resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); + + if (is_write) + res->obj->last_write = (VkAccessFlags)imb.dstAccessMask; + + res->obj->access = (VkAccessFlags)imb.dstAccessMask; + res->obj->access_stage = pipeline; + res->layout = new_layout; + if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + zink_resource_copies_reset(res); +} + +void +zink_resource_image_barrier2(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + VkImageMemoryBarrier2 imb; + if (!pipeline) + pipeline = pipeline_dst_stage(new_layout); + + if (!zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline)) + return; + bool is_write = zink_resource_access_is_write((VkAccessFlags)imb.dstAccessMask); + VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + assert(new_layout); + enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; + if (!res->obj->access_stage || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw)) + imb.srcAccessMask = 0; + if (res->obj->needs_zs_evaluate) + imb.pNext = &res->obj->zs_evaluate; + res->obj->needs_zs_evaluate = false; + if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) { + imb.srcQueueFamilyIndex = res->queue; + imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; + res->queue = VK_QUEUE_FAMILY_IGNORED; + } + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 0, + NULL, + 0, + NULL, + 1, + &imb + }; + bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout)); + VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep); + zink_cmd_debug_marker_end(ctx, cmdbuf, marker); + + resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); + + if (is_write) + res->obj->last_write = (VkAccessFlags)imb.dstAccessMask; + + res->obj->access = (VkAccessFlags)imb.dstAccessMask; + res->obj->access_stage = pipeline; + res->layout = new_layout; + if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) + zink_resource_copies_reset(res); +} + +bool +zink_check_unordered_transfer_access(struct zink_resource *res, unsigned level, const struct pipe_box *box) +{ + /* always barrier against previous non-transfer writes */ + bool non_transfer_write = res->obj->last_write && res->obj->last_write != VK_ACCESS_TRANSFER_WRITE_BIT; + /* must barrier if clobbering a previous write */ + bool transfer_clobber = res->obj->last_write == VK_ACCESS_TRANSFER_WRITE_BIT && zink_resource_copy_box_intersects(res, level, box); + return non_transfer_write || transfer_clobber; +} + +bool +zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) +{ + return res->obj->access && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !unordered_res_exec(ctx, res, false); +} + +void +zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box) +{ + if (res->obj->copies_need_reset) + zink_resource_copies_reset(res); + /* skip TRANSFER_DST barrier if no intersection from previous copies */ + if (res->layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL || + zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || + zink_check_unordered_transfer_access(res, level, box)) { + zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + } else { + res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; + res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT; + res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + } + zink_resource_copy_box_add(res, level, box); +} + +bool +zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size) +{ + if (res->obj->copies_need_reset) + zink_resource_copies_reset(res); + bool unordered = true; + struct pipe_box box = {(int)offset, 0, 0, (int)size, 0, 0}; + bool can_unordered_write = unordered_res_exec(ctx, res, true); + /* must barrier if something read the valid buffer range */ + bool valid_read = (res->obj->access || res->obj->unordered_access) && + util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write; + if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) { + zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + unordered = res->obj->unordered_write; + } else { + res->obj->unordered_access = VK_ACCESS_TRANSFER_WRITE_BIT; + res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT; + res->obj->unordered_access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + + ctx->batch.state->unordered_write_access |= VK_ACCESS_TRANSFER_WRITE_BIT; + ctx->batch.state->unordered_write_stages |= VK_PIPELINE_STAGE_TRANSFER_BIT; + if (!zink_resource_usage_matches(res, ctx->batch.state)) { + res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT; + res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + res->obj->ordered_access_is_copied = true; + } + } + zink_resource_copy_box_add(res, 0, &box); + /* this return value implies that the caller could do an unordered op on this resource */ + return unordered; +} + +VkPipelineStageFlags +zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage) +{ + switch (stage) { + case VK_SHADER_STAGE_VERTEX_BIT: + return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; + case VK_SHADER_STAGE_FRAGMENT_BIT: + return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + case VK_SHADER_STAGE_GEOMETRY_BIT: + return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT; + case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT: + return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT; + case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT: + return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT; + case VK_SHADER_STAGE_COMPUTE_BIT: + return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + default: + unreachable("unknown shader stage bit"); + } +} + +ALWAYS_INLINE static VkPipelineStageFlags +pipeline_access_stage(VkAccessFlags flags) +{ + if (flags & (VK_ACCESS_UNIFORM_READ_BIT | + VK_ACCESS_SHADER_READ_BIT | + VK_ACCESS_SHADER_WRITE_BIT)) + return VK_PIPELINE_STAGE_TASK_SHADER_BIT_NV | + VK_PIPELINE_STAGE_MESH_SHADER_BIT_NV | + VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR | + VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | + VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | + VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | + VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + return VK_PIPELINE_STAGE_TRANSFER_BIT; +} + +ALWAYS_INLINE static bool +buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline, bool unordered) +{ + return zink_resource_access_is_write(unordered ? res->obj->unordered_access : res->obj->access) || + zink_resource_access_is_write(flags) || + ((unordered ? res->obj->unordered_access_stage : res->obj->access_stage) & pipeline) != pipeline || + ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags; +} + +void +zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + if (!pipeline) + pipeline = pipeline_access_stage(flags); + + bool is_write = zink_resource_access_is_write(flags); + bool unordered = unordered_res_exec(ctx, res, is_write); + if (!buffer_needs_barrier(res, flags, pipeline, unordered)) + return; + enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; + bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); + bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); + bool unordered_usage_matches = res->obj->unordered_access && usage_matches; + if (completed) { + /* reset access on complete */ + res->obj->access = VK_ACCESS_NONE; + res->obj->access_stage = VK_PIPELINE_STAGE_NONE; + res->obj->last_write = VK_ACCESS_NONE; + } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) { + /* always reset propagated access to avoid weirdness */ + res->obj->access = VK_ACCESS_NONE; + res->obj->access_stage = VK_PIPELINE_STAGE_NONE; + } else if (!unordered && !unordered_usage_matches) { + /* reset unordered access on first ordered barrier */ + res->obj->unordered_access = VK_ACCESS_NONE; + res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; + } + if (!usage_matches) { + /* reset unordered on first new cmdbuf barrier */ + res->obj->unordered_access = VK_ACCESS_NONE; + res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; + res->obj->ordered_access_is_copied = false; + } + /* unordered barriers can be skipped if either: + * - there is no current-batch unordered access + * - the unordered access is not write access + */ + bool can_skip_unordered = !unordered ? false : (!unordered_usage_matches || !zink_resource_access_is_write(res->obj->unordered_access)); + /* ordered barriers can be skipped if both: + * - there is no current access + * - there is no current-batch unordered access + */ + bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches); + + if (!can_skip_unordered && !can_skip_ordered) { + VkMemoryBarrier bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + bmb.pNext = NULL; + VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; + if (unordered) { + stages = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcAccessMask = res->obj->access; + } + + VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + bool marker = false; + if (unlikely(zink_tracing)) { + char buf[4096]; + bool first = true; + unsigned idx = 0; + u_foreach_bit64(bit, flags) { + if (!first) + buf[idx++] = '|'; + idx += snprintf(&buf[idx], sizeof(buf) - idx, "%s", vk_AccessFlagBits_to_str((VkAccessFlagBits)(1ul<obj->last_write = flags; + if (unordered) { + /* these should get automatically emitted during submission */ + res->obj->unordered_access = flags; + res->obj->unordered_access_stage = pipeline; + if (is_write) { + ctx->batch.state->unordered_write_access |= flags; + ctx->batch.state->unordered_write_stages |= pipeline; + } + } + if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) { + res->obj->access = flags; + res->obj->access_stage = pipeline; + res->obj->ordered_access_is_copied = unordered; + } + if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write) + zink_resource_copies_reset(res); +} + +void +zink_resource_buffer_barrier2(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) +{ + if (!pipeline) + pipeline = pipeline_access_stage(flags); + + bool is_write = zink_resource_access_is_write(flags); + bool unordered = unordered_res_exec(ctx, res, is_write); + if (!buffer_needs_barrier(res, flags, pipeline, unordered)) + return; + enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE; + bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw); + bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state); + bool unordered_usage_matches = res->obj->unordered_access && usage_matches; + if (completed) { + /* reset access on complete */ + res->obj->access = VK_ACCESS_NONE; + res->obj->access_stage = VK_PIPELINE_STAGE_NONE; + res->obj->last_write = VK_ACCESS_NONE; + } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) { + /* always reset propagated access to avoid weirdness */ + res->obj->access = VK_ACCESS_NONE; + res->obj->access_stage = VK_PIPELINE_STAGE_NONE; + } else if (!unordered && !unordered_usage_matches) { + /* reset unordered access on first ordered barrier */ + res->obj->unordered_access = VK_ACCESS_NONE; + res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; + } + if (!usage_matches) { + /* reset unordered on first new cmdbuf barrier */ + res->obj->unordered_access = VK_ACCESS_NONE; + res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE; + res->obj->ordered_access_is_copied = false; + } + /* unordered barriers can be skipped if either: + * - there is no current-batch unordered access + * - the unordered access is not write access + */ + bool can_skip_unordered = !unordered ? false : (!unordered_usage_matches || !zink_resource_access_is_write(res->obj->unordered_access)); + /* ordered barriers can be skipped if both: + * - there is no current access + * - there is no current-batch unordered access + */ + bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches); + + if (!can_skip_unordered && !can_skip_ordered) { + VkMemoryBarrier2 bmb; + bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; + bmb.pNext = NULL; + VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);; + if (unordered) { + bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages; + bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access; + } else { + bmb.srcStageMask = stages; + bmb.srcAccessMask = res->obj->access; + } + bmb.dstStageMask = pipeline; + bmb.dstAccessMask = flags; + VkDependencyInfo dep = { + VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + NULL, + 0, + 1, + &bmb, + 0, + NULL, + 0, + NULL + }; + VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); + bool marker = false; + if (unlikely(zink_tracing)) { + char buf[4096]; + bool first = true; + unsigned idx = 0; + u_foreach_bit64(bit, flags) { + if (!first) + buf[idx++] = '|'; + idx += snprintf(&buf[idx], sizeof(buf) - idx, "%s", vk_AccessFlagBits_to_str((VkAccessFlagBits)(1ul<obj->last_write = flags; + if (unordered) { + /* these should get automatically emitted during submission */ + res->obj->unordered_access = flags; + res->obj->unordered_access_stage = pipeline; + if (is_write) { + ctx->batch.state->unordered_write_access |= flags; + ctx->batch.state->unordered_write_stages |= pipeline; + } + } + if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) { + res->obj->access = flags; + res->obj->access_stage = pipeline; + res->obj->ordered_access_is_copied = unordered; + } + if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write) + zink_resource_copies_reset(res); +}