src/gallium/drivers/zink/zink_context.c

   1 /*
   2  * Copyright 2018 Collabora Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "zink_clear.h"
  25 #include "zink_context.h"
  26 #include "zink_descriptors.h"
  27 #include "zink_fence.h"
  28 #include "zink_format.h"
  29 #include "zink_framebuffer.h"
  30 #include "zink_helpers.h"
  31 #include "zink_inlines.h"
  32 #include "zink_kopper.h"
  33 #include "zink_program.h"
  34 #include "zink_query.h"
  35 #include "zink_render_pass.h"
  36 #include "zink_resource.h"
  37 #include "zink_screen.h"
  38 #include "zink_state.h"
  39 #include "zink_surface.h"
  40
  41
  42 #include "util/u_blitter.h"
  43 #include "util/u_debug.h"
  44 #include "util/format_srgb.h"
  45 #include "util/format/u_format.h"
  46 #include "util/u_helpers.h"
  47 #include "util/u_inlines.h"
  48 #include "util/u_thread.h"
  49 #include "util/u_cpu_detect.h"
  50 #include "util/strndup.h"
  51 #include "nir.h"
  52
  53 #include "driver_trace/tr_context.h"
  54
  55 #include "util/u_memory.h"
  56 #include "util/u_upload_mgr.h"
  57
  58 #define XXH_INLINE_ALL
  59 #include "util/xxhash.h"
  60
  61 static void
  62 update_tc_info(struct zink_context *ctx, bool wait)
  63 {
  64    if (ctx->tc) {
  65       const struct tc_renderpass_info *info = threaded_context_get_renderpass_info(ctx->tc, wait);
  66       if (info) {
  67          ctx->rp_changed |= ctx->dynamic_fb.tc_info.data != info->data;
  68          ctx->dynamic_fb.tc_info.data = info->data;
  69       }
  70    }
  71 }
  72
  73 void
  74 debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr)
  75 {
  76    sprintf(buf, "zink_buffer_view");
  77 }
  78
  79 ALWAYS_INLINE static void
  80 check_resource_for_batch_ref(struct zink_context *ctx, struct zink_resource *res)
  81 {
  82    if (!zink_resource_has_binds(res)) {
  83       /* avoid desync between usage and tracking:
  84        * - if usage exists, it must be removed before the context is destroyed
  85        * - having usage does not imply having tracking
  86        * - if tracking will be added here, also reapply usage to avoid dangling usage once tracking is removed
  87        * TODO: somehow fix this for perf because it's an extra hash lookup
  88        */
  89       if (!res->obj->dt && (res->obj->bo->reads || res->obj->bo->writes))
  90          zink_batch_reference_resource_rw(&ctx->batch, res, !!res->obj->bo->writes);
  91       else
  92          zink_batch_reference_resource(&ctx->batch, res);
  93    }
  94 }
  95
  96 static void
  97 zink_context_destroy(struct pipe_context *pctx)
  98 {
  99    struct zink_context *ctx = zink_context(pctx);
 100    struct zink_screen *screen = zink_screen(pctx->screen);
 101
 102    if (util_queue_is_initialized(&screen->flush_queue))
 103       util_queue_finish(&screen->flush_queue);
 104    if (ctx->batch.state && !screen->device_lost) {
 105       VkResult result = VKSCR(QueueWaitIdle)(screen->queue);
 106
 107       if (result != VK_SUCCESS)
 108          mesa_loge("ZINK: vkQueueWaitIdle failed (%s)", vk_Result_to_str(result));
 109    }
 110
 111    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) {
 112       simple_mtx_lock((&ctx->program_lock[i]));
 113       hash_table_foreach(&ctx->program_cache[i], entry) {
 114          struct zink_program *pg = entry->data;
 115          pg->removed = true;
 116       }
 117       simple_mtx_unlock((&ctx->program_lock[i]));
 118    }
 119
 120    if (ctx->blitter)
 121       util_blitter_destroy(ctx->blitter);
 122    for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
 123       pipe_surface_release(&ctx->base, &ctx->fb_state.cbufs[i]);
 124    pipe_surface_release(&ctx->base, &ctx->fb_state.zsbuf);
 125
 126    pipe_resource_reference(&ctx->dummy_vertex_buffer, NULL);
 127    pipe_resource_reference(&ctx->dummy_xfb_buffer, NULL);
 128
 129    for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++)
 130       pipe_surface_release(&ctx->base, &ctx->dummy_surface[i]);
 131    zink_buffer_view_reference(screen, &ctx->dummy_bufferview, NULL);
 132
 133    zink_descriptors_deinit_bindless(ctx);
 134
 135    if (ctx->batch.state) {
 136       zink_clear_batch_state(ctx, ctx->batch.state);
 137       zink_batch_state_destroy(screen, ctx->batch.state);
 138    }
 139    struct zink_batch_state *bs = ctx->batch_states;
 140    while (bs) {
 141       struct zink_batch_state *bs_next = bs->next;
 142       zink_clear_batch_state(ctx, bs);
 143       zink_batch_state_destroy(screen, bs);
 144       bs = bs_next;
 145    }
 146    bs = ctx->free_batch_states;
 147    while (bs) {
 148       struct zink_batch_state *bs_next = bs->next;
 149       zink_clear_batch_state(ctx, bs);
 150       zink_batch_state_destroy(screen, bs);
 151       bs = bs_next;
 152    }
 153
 154    for (unsigned i = 0; i < 2; i++) {
 155       util_idalloc_fini(&ctx->di.bindless[i].tex_slots);
 156       util_idalloc_fini(&ctx->di.bindless[i].img_slots);
 157       free(ctx->di.bindless[i].buffer_infos);
 158       free(ctx->di.bindless[i].img_infos);
 159       util_dynarray_fini(&ctx->di.bindless[i].updates);
 160       util_dynarray_fini(&ctx->di.bindless[i].resident);
 161    }
 162
 163    hash_table_foreach(&ctx->framebuffer_cache, he)
 164       zink_destroy_framebuffer(screen, he->data);
 165
 166    hash_table_foreach(ctx->render_pass_cache, he)
 167       zink_destroy_render_pass(screen, he->data);
 168
 169    zink_context_destroy_query_pools(ctx);
 170    u_upload_destroy(pctx->stream_uploader);
 171    u_upload_destroy(pctx->const_uploader);
 172    slab_destroy_child(&ctx->transfer_pool);
 173    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++)
 174       _mesa_hash_table_clear(&ctx->program_cache[i], NULL);
 175    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_lock); i++)
 176       simple_mtx_destroy(&ctx->program_lock[i]);
 177    _mesa_hash_table_destroy(ctx->render_pass_cache, NULL);
 178    slab_destroy_child(&ctx->transfer_pool_unsync);
 179
 180    zink_descriptors_deinit(ctx);
 181
 182    if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY))
 183       p_atomic_dec(&screen->base.num_contexts);
 184
 185    ralloc_free(ctx);
 186 }
 187
 188 static void
 189 check_device_lost(struct zink_context *ctx)
 190 {
 191    if (!zink_screen(ctx->base.screen)->device_lost || ctx->is_device_lost)
 192       return;
 193    debug_printf("ZINK: device lost detected!\n");
 194    if (ctx->reset.reset)
 195       ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
 196    ctx->is_device_lost = true;
 197 }
 198
 199 static enum pipe_reset_status
 200 zink_get_device_reset_status(struct pipe_context *pctx)
 201 {
 202    struct zink_context *ctx = zink_context(pctx);
 203
 204    enum pipe_reset_status status = PIPE_NO_RESET;
 205
 206    if (ctx->is_device_lost) {
 207       // Since we don't know what really happened to the hardware, just
 208       // assume that we are in the wrong
 209       status = PIPE_GUILTY_CONTEXT_RESET;
 210
 211       debug_printf("ZINK: device lost detected!\n");
 212
 213       if (ctx->reset.reset)
 214          ctx->reset.reset(ctx->reset.data, status);
 215    }
 216
 217    return status;
 218 }
 219
 220 static void
 221 zink_set_device_reset_callback(struct pipe_context *pctx,
 222                                const struct pipe_device_reset_callback *cb)
 223 {
 224    struct zink_context *ctx = zink_context(pctx);
 225    bool had_reset = !!ctx->reset.reset;
 226
 227    if (cb)
 228       ctx->reset = *cb;
 229    else
 230       memset(&ctx->reset, 0, sizeof(ctx->reset));
 231
 232    bool have_reset = !!ctx->reset.reset;
 233    if (had_reset != have_reset) {
 234       if (have_reset)
 235          p_atomic_inc(&zink_screen(pctx->screen)->robust_ctx_count);
 236       else
 237          p_atomic_dec(&zink_screen(pctx->screen)->robust_ctx_count);
 238    }
 239 }
 240
 241 static void
 242 zink_set_context_param(struct pipe_context *pctx, enum pipe_context_param param,
 243                        unsigned value)
 244 {
 245    struct zink_context *ctx = zink_context(pctx);
 246
 247    switch (param) {
 248    case PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE:
 249       util_set_thread_affinity(zink_screen(ctx->base.screen)->flush_queue.threads[0],
 250                                util_get_cpu_caps()->L3_affinity_mask[value],
 251                                NULL, util_get_cpu_caps()->num_cpu_mask_bits);
 252       break;
 253    default:
 254       break;
 255    }
 256 }
 257
 258 static VkSamplerMipmapMode
 259 sampler_mipmap_mode(enum pipe_tex_mipfilter filter)
 260 {
 261    switch (filter) {
 262    case PIPE_TEX_MIPFILTER_NEAREST: return VK_SAMPLER_MIPMAP_MODE_NEAREST;
 263    case PIPE_TEX_MIPFILTER_LINEAR: return VK_SAMPLER_MIPMAP_MODE_LINEAR;
 264    case PIPE_TEX_MIPFILTER_NONE:
 265       unreachable("PIPE_TEX_MIPFILTER_NONE should be dealt with earlier");
 266    }
 267    unreachable("unexpected filter");
 268 }
 269
 270 static VkSamplerAddressMode
 271 sampler_address_mode(enum pipe_tex_wrap filter)
 272 {
 273    switch (filter) {
 274    case PIPE_TEX_WRAP_REPEAT: return VK_SAMPLER_ADDRESS_MODE_REPEAT;
 275    case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
 276    case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
 277    case PIPE_TEX_WRAP_MIRROR_REPEAT: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
 278    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE;
 279    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; /* not technically correct, but kinda works */
 280    default: break;
 281    }
 282    unreachable("unexpected wrap");
 283 }
 284
 285 static VkCompareOp
 286 compare_op(enum pipe_compare_func op)
 287 {
 288    switch (op) {
 289       case PIPE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
 290       case PIPE_FUNC_LESS: return VK_COMPARE_OP_LESS;
 291       case PIPE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
 292       case PIPE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
 293       case PIPE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
 294       case PIPE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
 295       case PIPE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
 296       case PIPE_FUNC_ALWAYS: return VK_COMPARE_OP_ALWAYS;
 297    }
 298    unreachable("unexpected compare");
 299 }
 300
 301 static inline bool
 302 wrap_needs_border_color(unsigned wrap)
 303 {
 304    return wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
 305           wrap == PIPE_TEX_WRAP_MIRROR_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER;
 306 }
 307
 308 static VkBorderColor
 309 get_border_color(const union pipe_color_union *color, bool is_integer, bool need_custom)
 310 {
 311    if (is_integer) {
 312       if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 0)
 313          return VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
 314       if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 1)
 315          return VK_BORDER_COLOR_INT_OPAQUE_BLACK;
 316       if (color->ui[0] == 1 && color->ui[1] == 1 && color->ui[2] == 1 && color->ui[3] == 1)
 317          return VK_BORDER_COLOR_INT_OPAQUE_WHITE;
 318       return need_custom ? VK_BORDER_COLOR_INT_CUSTOM_EXT : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
 319    }
 320
 321    if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 0)
 322       return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 323    if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 1)
 324       return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
 325    if (color->f[0] == 1 && color->f[1] == 1 && color->f[2] == 1 && color->f[3] == 1)
 326       return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
 327    return need_custom ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 328 }
 329
 330 static void *
 331 zink_create_sampler_state(struct pipe_context *pctx,
 332                           const struct pipe_sampler_state *state)
 333 {
 334    struct zink_screen *screen = zink_screen(pctx->screen);
 335    bool need_custom = false;
 336    bool need_clamped_border_color = false;
 337    VkSamplerCreateInfo sci = {0};
 338    VkSamplerCustomBorderColorCreateInfoEXT cbci = {0};
 339    VkSamplerCustomBorderColorCreateInfoEXT cbci_clamped = {0};
 340    sci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
 341    if (screen->info.have_EXT_non_seamless_cube_map && !state->seamless_cube_map)
 342       sci.flags |= VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT;
 343    assert(!state->unnormalized_coords);
 344    sci.magFilter = zink_filter(state->mag_img_filter);
 345    if (sci.unnormalizedCoordinates)
 346       sci.minFilter = sci.magFilter;
 347    else
 348       sci.minFilter = zink_filter(state->min_img_filter);
 349
 350    VkSamplerReductionModeCreateInfo rci;
 351    rci.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO;
 352    rci.pNext = NULL;
 353    switch (state->reduction_mode) {
 354    case PIPE_TEX_REDUCTION_MIN:
 355       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MIN;
 356       break;
 357    case PIPE_TEX_REDUCTION_MAX:
 358       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX;
 359       break;
 360    default:
 361       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE;
 362       break;
 363    }
 364    if (state->reduction_mode)
 365       sci.pNext = &rci;
 366
 367    if (sci.unnormalizedCoordinates) {
 368       sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
 369    } else if (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE) {
 370       sci.mipmapMode = sampler_mipmap_mode(state->min_mip_filter);
 371       sci.minLod = state->min_lod;
 372       sci.maxLod = MAX2(state->max_lod, state->min_lod);
 373    } else {
 374       sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
 375       sci.minLod = 0;
 376       sci.maxLod = 0.25f;
 377    }
 378
 379    if (!sci.unnormalizedCoordinates) {
 380       sci.addressModeU = sampler_address_mode(state->wrap_s);
 381       sci.addressModeV = sampler_address_mode(state->wrap_t);
 382       sci.addressModeW = sampler_address_mode(state->wrap_r);
 383    } else {
 384       sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
 385    }
 386
 387    sci.mipLodBias = CLAMP(state->lod_bias,
 388                           -screen->info.props.limits.maxSamplerLodBias,
 389                           screen->info.props.limits.maxSamplerLodBias);
 390
 391    need_custom |= wrap_needs_border_color(state->wrap_s);
 392    need_custom |= wrap_needs_border_color(state->wrap_t);
 393    need_custom |= wrap_needs_border_color(state->wrap_r);
 394
 395    if (state->compare_mode == PIPE_TEX_COMPARE_NONE)
 396       sci.compareOp = VK_COMPARE_OP_NEVER;
 397    else {
 398       sci.compareOp = compare_op(state->compare_func);
 399       sci.compareEnable = VK_TRUE;
 400    }
 401
 402    bool is_integer = state->border_color_is_integer;
 403
 404    sci.borderColor = get_border_color(&state->border_color, is_integer, need_custom);
 405    if (sci.borderColor > VK_BORDER_COLOR_INT_OPAQUE_WHITE && need_custom) {
 406       if (!screen->info.border_color_feats.customBorderColorWithoutFormat &&
 407           screen->info.driver_props.driverID != VK_DRIVER_ID_MESA_TURNIP) {
 408          static bool warned = false;
 409          warn_missing_feature(warned, "customBorderColorWithoutFormat");
 410       }
 411       if (screen->info.have_EXT_custom_border_color &&
 412           (screen->info.border_color_feats.customBorderColorWithoutFormat || state->border_color_format)) {
 413          if (!screen->info.have_EXT_border_color_swizzle) {
 414             static bool warned = false;
 415             warn_missing_feature(warned, "VK_EXT_border_color_swizzle");
 416          }
 417
 418          if (!is_integer && !screen->have_D24_UNORM_S8_UINT) {
 419             union pipe_color_union clamped_border_color;
 420             for (unsigned i = 0; i < 4; ++i) {
 421                /* Use channel 0 on purpose, so that we can use OPAQUE_WHITE
 422                 * when the border color is 1.0. */
 423                clamped_border_color.f[i] = CLAMP(state->border_color.f[0], 0, 1);
 424             }
 425             if (memcmp(&state->border_color, &clamped_border_color, sizeof(clamped_border_color)) != 0) {
 426                need_clamped_border_color = true;
 427                cbci_clamped.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
 428                cbci_clamped.format = VK_FORMAT_UNDEFINED;
 429                /* these are identical unions */
 430                memcpy(&cbci_clamped.customBorderColor, &clamped_border_color, sizeof(union pipe_color_union));
 431             }
 432          }
 433          cbci.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
 434          if (screen->info.border_color_feats.customBorderColorWithoutFormat) {
 435             cbci.format = VK_FORMAT_UNDEFINED;
 436             /* these are identical unions */
 437             memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
 438          } else {
 439             if (util_format_is_depth_or_stencil(state->border_color_format)) {
 440                if (is_integer) {
 441                   cbci.format = VK_FORMAT_S8_UINT;
 442                   for (unsigned i = 0; i < 4; i++)
 443                      cbci.customBorderColor.uint32[i] = CLAMP(state->border_color.ui[i], 0, 255);
 444                } else {
 445                   cbci.format = zink_get_format(screen, util_format_get_depth_only(state->border_color_format));
 446                   /* these are identical unions */
 447                   memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
 448                }
 449             } else {
 450                cbci.format = zink_get_format(screen, state->border_color_format);
 451                for (unsigned i = 0; i < 4; i++) {
 452                   zink_format_clamp_channel_color(util_format_description(state->border_color_format), (void*)&cbci.customBorderColor, &state->border_color, i);
 453                   zink_format_clamp_channel_srgb(util_format_description(state->border_color_format), (void*)&cbci.customBorderColor, (void*)&cbci.customBorderColor, i);
 454                }
 455             }
 456          }
 457          cbci.pNext = sci.pNext;
 458          sci.pNext = &cbci;
 459          UNUSED uint32_t check = p_atomic_inc_return(&screen->cur_custom_border_color_samplers);
 460          assert(check <= screen->info.border_color_props.maxCustomBorderColorSamplers);
 461       } else
 462          sci.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; // TODO with custom shader if we're super interested?
 463       if (sci.unnormalizedCoordinates)
 464          sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
 465    }
 466
 467    if (state->max_anisotropy > 1) {
 468       sci.maxAnisotropy = state->max_anisotropy;
 469       sci.anisotropyEnable = VK_TRUE;
 470    }
 471
 472    struct zink_sampler_state *sampler = CALLOC_STRUCT(zink_sampler_state);
 473    if (!sampler)
 474       return NULL;
 475
 476    VkResult result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler);
 477    if (result != VK_SUCCESS) {
 478       mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
 479       FREE(sampler);
 480       return NULL;
 481    }
 482    if (need_clamped_border_color) {
 483       sci.pNext = &cbci_clamped;
 484       result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler_clamped);
 485       if (result != VK_SUCCESS) {
 486          mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
 487          VKSCR(DestroySampler)(screen->dev, sampler->sampler, NULL);
 488          FREE(sampler);
 489          return NULL;
 490       }
 491    }
 492    sampler->custom_border_color = need_custom;
 493    if (!screen->info.have_EXT_non_seamless_cube_map)
 494       sampler->emulate_nonseamless = !state->seamless_cube_map;
 495
 496    return sampler;
 497 }
 498
 499 ALWAYS_INLINE static VkImageLayout
 500 get_layout_for_binding(const struct zink_context *ctx, struct zink_resource *res, enum zink_descriptor_type type, bool is_compute)
 501 {
 502    if (res->obj->is_buffer)
 503       return 0;
 504    switch (type) {
 505    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
 506       return zink_descriptor_util_image_layout_eval(ctx, res, is_compute);
 507    case ZINK_DESCRIPTOR_TYPE_IMAGE:
 508       return VK_IMAGE_LAYOUT_GENERAL;
 509    default:
 510       break;
 511    }
 512    return 0;
 513 }
 514
 515 ALWAYS_INLINE static struct zink_surface *
 516 get_imageview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
 517 {
 518    switch (type) {
 519    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
 520       struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
 521       if (!sampler_view || !sampler_view->base.texture)
 522          return NULL;
 523       /* if this is a non-seamless cube sampler, return the cube array view */
 524       return (ctx->di.emulate_nonseamless[stage] & ctx->di.cubes[stage] & BITFIELD_BIT(idx)) ?
 525              sampler_view->cube_array :
 526              sampler_view->image_view;
 527    }
 528    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
 529       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
 530       return image_view->base.resource ? image_view->surface : NULL;
 531    }
 532    default:
 533       break;
 534    }
 535    unreachable("ACK");
 536    return VK_NULL_HANDLE;
 537 }
 538
 539 ALWAYS_INLINE static struct zink_buffer_view *
 540 get_bufferview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
 541 {
 542    switch (type) {
 543    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
 544       struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
 545       return sampler_view->base.texture ? sampler_view->buffer_view : NULL;
 546    }
 547    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
 548       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
 549       return image_view->base.resource ? image_view->buffer_view : NULL;
 550    }
 551    default:
 552       break;
 553    }
 554    unreachable("ACK");
 555    return VK_NULL_HANDLE;
 556 }
 557
 558 ALWAYS_INLINE static struct zink_resource *
 559 update_descriptor_state_ubo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 560 {
 561    struct zink_screen *screen = zink_screen(ctx->base.screen);
 562    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 563    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_UBO;
 564    ctx->di.descriptor_res[type][shader][slot] = res;
 565    ctx->di.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset;
 566    if (res) {
 567       ctx->di.ubos[shader][slot].buffer = res->obj->buffer;
 568       ctx->di.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size;
 569       assert(ctx->di.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
 570    } else {
 571       VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
 572       ctx->di.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
 573       ctx->di.ubos[shader][slot].range = VK_WHOLE_SIZE;
 574    }
 575    if (!slot) {
 576       if (res)
 577          ctx->di.push_valid |= BITFIELD64_BIT(shader);
 578       else
 579          ctx->di.push_valid &= ~BITFIELD64_BIT(shader);
 580    }
 581    return res;
 582 }
 583
 584 ALWAYS_INLINE static struct zink_resource *
 585 update_descriptor_state_ssbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 586 {
 587    struct zink_screen *screen = zink_screen(ctx->base.screen);
 588    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 589    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SSBO;
 590    ctx->di.descriptor_res[type][shader][slot] = res;
 591    ctx->di.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset;
 592    if (res) {
 593       ctx->di.ssbos[shader][slot].buffer = res->obj->buffer;
 594       ctx->di.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size;
 595    } else {
 596       VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
 597       ctx->di.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
 598       ctx->di.ssbos[shader][slot].range = VK_WHOLE_SIZE;
 599    }
 600    return res;
 601 }
 602
 603 ALWAYS_INLINE static struct zink_resource *
 604 update_descriptor_state_sampler(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 605 {
 606    struct zink_screen *screen = zink_screen(ctx->base.screen);
 607    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 608    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW;
 609    ctx->di.descriptor_res[type][shader][slot] = res;
 610    if (res) {
 611       if (res->obj->is_buffer) {
 612          struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
 613          ctx->di.tbos[shader][slot] = bv->buffer_view;
 614       } else {
 615          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
 616          ctx->di.textures[shader][slot].imageLayout = get_layout_for_binding(ctx, res, type, shader == MESA_SHADER_COMPUTE);
 617          ctx->di.textures[shader][slot].imageView = surface->image_view;
 618          if (!screen->have_D24_UNORM_S8_UINT &&
 619              ctx->sampler_states[shader][slot] && ctx->sampler_states[shader][slot]->sampler_clamped) {
 620             struct zink_sampler_state *state = ctx->sampler_states[shader][slot];
 621             VkSampler sampler = (surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
 622                                 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT) ?
 623                                 state->sampler_clamped :
 624                                 state->sampler;
 625             if (ctx->di.textures[shader][slot].sampler != sampler) {
 626                zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
 627                ctx->di.textures[shader][slot].sampler = sampler;
 628             }
 629          }
 630       }
 631    } else {
 632       if (likely(have_null_descriptors)) {
 633          ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE;
 634          ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
 635          ctx->di.tbos[shader][slot] = VK_NULL_HANDLE;
 636       } else {
 637          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
 638          struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
 639          ctx->di.textures[shader][slot].imageView = null_surface->image_view;
 640          ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 641          ctx->di.tbos[shader][slot] = null_bufferview->buffer_view;
 642       }
 643    }
 644    return res;
 645 }
 646
 647 ALWAYS_INLINE static struct zink_resource *
 648 update_descriptor_state_image(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 649 {
 650    struct zink_screen *screen = zink_screen(ctx->base.screen);
 651    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 652    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_IMAGE;
 653    ctx->di.descriptor_res[type][shader][slot] = res;
 654    if (res) {
 655       if (res->obj->is_buffer) {
 656          struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
 657          ctx->di.texel_images[shader][slot] = bv->buffer_view;
 658       } else {
 659          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
 660          ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
 661          ctx->di.images[shader][slot].imageView = surface->image_view;
 662       }
 663    } else {
 664       if (likely(have_null_descriptors)) {
 665          memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot]));
 666          ctx->di.texel_images[shader][slot] = VK_NULL_HANDLE;
 667       } else {
 668          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
 669          struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
 670          ctx->di.images[shader][slot].imageView = null_surface->image_view;
 671          ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
 672          ctx->di.texel_images[shader][slot] = null_bufferview->buffer_view;
 673       }
 674    }
 675    return res;
 676 }
 677
 678 static void
 679 update_nonseamless_shader_key(struct zink_context *ctx, gl_shader_stage pstage)
 680 {
 681    const uint32_t new_mask = ctx->di.emulate_nonseamless[pstage] & ctx->di.cubes[pstage];
 682    if (pstage == MESA_SHADER_COMPUTE) {
 683       if (ctx->compute_pipeline_state.key.base.nonseamless_cube_mask != new_mask)
 684          ctx->compute_dirty = true;
 685       ctx->compute_pipeline_state.key.base.nonseamless_cube_mask = new_mask;
 686    } else {
 687       if (zink_get_shader_key_base(ctx, pstage)->nonseamless_cube_mask != new_mask)
 688          zink_set_shader_key_base(ctx, pstage)->nonseamless_cube_mask = new_mask;
 689    }
 690 }
 691
 692 static void
 693 zink_bind_sampler_states(struct pipe_context *pctx,
 694                          gl_shader_stage shader,
 695                          unsigned start_slot,
 696                          unsigned num_samplers,
 697                          void **samplers)
 698 {
 699    struct zink_context *ctx = zink_context(pctx);
 700    struct zink_screen *screen = zink_screen(pctx->screen);
 701    for (unsigned i = 0; i < num_samplers; ++i) {
 702       struct zink_sampler_state *state = samplers[i];
 703       if (ctx->sampler_states[shader][start_slot + i] != state)
 704          zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, 1);
 705       ctx->sampler_states[shader][start_slot + i] = state;
 706       if (state) {
 707          ctx->di.textures[shader][start_slot + i].sampler = state->sampler;
 708          if (state->sampler_clamped && !screen->have_D24_UNORM_S8_UINT) {
 709             struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
 710             if (surface &&
 711                 ((surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
 712                  (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT)))
 713                ctx->di.textures[shader][start_slot + i].sampler = state->sampler_clamped;
 714          }
 715       } else {
 716          ctx->di.textures[shader][start_slot + i].sampler = VK_NULL_HANDLE;
 717       }
 718    }
 719    ctx->di.num_samplers[shader] = start_slot + num_samplers;
 720 }
 721
 722 static void
 723 zink_bind_sampler_states_nonseamless(struct pipe_context *pctx,
 724                                      gl_shader_stage shader,
 725                                      unsigned start_slot,
 726                                      unsigned num_samplers,
 727                                      void **samplers)
 728 {
 729    struct zink_context *ctx = zink_context(pctx);
 730    uint32_t old_mask = ctx->di.emulate_nonseamless[shader];
 731    uint32_t mask = BITFIELD_RANGE(start_slot, num_samplers);
 732    ctx->di.emulate_nonseamless[shader] &= ~mask;
 733    for (unsigned i = 0; i < num_samplers; ++i) {
 734       struct zink_sampler_state *state = samplers[i];
 735       const uint32_t bit = BITFIELD_BIT(start_slot + i);
 736       if (!state)
 737          continue;
 738       if (state->emulate_nonseamless)
 739          ctx->di.emulate_nonseamless[shader] |= bit;
 740       if (state->emulate_nonseamless != (old_mask & bit) && (ctx->di.cubes[shader] & bit)) {
 741          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
 742          if (surface && ctx->di.images[shader][start_slot + i].imageView != surface->image_view) {
 743             ctx->di.images[shader][start_slot + i].imageView = surface->image_view;
 744             update_descriptor_state_sampler(ctx, shader, start_slot + i, zink_resource(surface->base.texture));
 745             zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i, 1);
 746          }
 747       }
 748    }
 749    zink_bind_sampler_states(pctx, shader, start_slot, num_samplers, samplers);
 750    update_nonseamless_shader_key(ctx, shader);
 751 }
 752
 753 static void
 754 zink_delete_sampler_state(struct pipe_context *pctx,
 755                           void *sampler_state)
 756 {
 757    struct zink_sampler_state *sampler = sampler_state;
 758    struct zink_batch *batch = &zink_context(pctx)->batch;
 759    /* may be called if context_create fails */
 760    if (batch->state) {
 761       util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
 762                            sampler->sampler);
 763       if (sampler->sampler_clamped)
 764          util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
 765                               sampler->sampler_clamped);
 766    }
 767    if (sampler->custom_border_color)
 768       p_atomic_dec(&zink_screen(pctx->screen)->cur_custom_border_color_samplers);
 769    FREE(sampler);
 770 }
 771
 772 static VkImageAspectFlags
 773 sampler_aspect_from_format(enum pipe_format fmt)
 774 {
 775    if (util_format_is_depth_or_stencil(fmt)) {
 776       const struct util_format_description *desc = util_format_description(fmt);
 777       if (util_format_has_depth(desc))
 778          return VK_IMAGE_ASPECT_DEPTH_BIT;
 779       assert(util_format_has_stencil(desc));
 780       return VK_IMAGE_ASPECT_STENCIL_BIT;
 781    } else
 782      return VK_IMAGE_ASPECT_COLOR_BIT;
 783 }
 784
 785 static uint32_t
 786 hash_bufferview(void *bvci)
 787 {
 788    size_t offset = offsetof(VkBufferViewCreateInfo, flags);
 789    return _mesa_hash_data((char*)bvci + offset, sizeof(VkBufferViewCreateInfo) - offset);
 790 }
 791
 792 static VkBufferViewCreateInfo
 793 create_bvci(struct zink_context *ctx, struct zink_resource *res, enum pipe_format format, uint32_t offset, uint32_t range)
 794 {
 795    struct zink_screen *screen = zink_screen(ctx->base.screen);
 796    VkBufferViewCreateInfo bvci;
 797    // Zero whole struct (including alignment holes), so hash_bufferview
 798    // does not access potentially uninitialized data.
 799    memset(&bvci, 0, sizeof(bvci));
 800    bvci.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO;
 801    bvci.pNext = NULL;
 802    if (screen->format_props[format].bufferFeatures & VK_FORMAT_FEATURE_STORAGE_TEXEL_BUFFER_BIT)
 803       bvci.buffer = res->obj->storage_buffer ? res->obj->storage_buffer : res->obj->buffer;
 804    else
 805       bvci.buffer = res->obj->buffer;
 806    bvci.format = zink_get_format(screen, format);
 807    assert(bvci.format);
 808    bvci.offset = offset;
 809    bvci.range = !offset && range == res->base.b.width0 ? VK_WHOLE_SIZE : range;
 810    unsigned blocksize = util_format_get_blocksize(format);
 811    if (bvci.range != VK_WHOLE_SIZE) {
 812       /* clamp out partial texels */
 813       bvci.range -= bvci.range % blocksize;
 814       if (bvci.offset + bvci.range >= res->base.b.width0)
 815          bvci.range = VK_WHOLE_SIZE;
 816    }
 817    uint64_t clamp = blocksize * screen->info.props.limits.maxTexelBufferElements;
 818    if (bvci.range == VK_WHOLE_SIZE && res->base.b.width0 > clamp)
 819       bvci.range = clamp;
 820    bvci.flags = 0;
 821    return bvci;
 822 }
 823
 824 static struct zink_buffer_view *
 825 get_buffer_view(struct zink_context *ctx, struct zink_resource *res, VkBufferViewCreateInfo *bvci)
 826 {
 827    struct zink_screen *screen = zink_screen(ctx->base.screen);
 828    struct zink_buffer_view *buffer_view = NULL;
 829
 830    uint32_t hash = hash_bufferview(bvci);
 831    simple_mtx_lock(&res->bufferview_mtx);
 832    struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, hash, bvci);
 833    if (he) {
 834       buffer_view = he->data;
 835       p_atomic_inc(&buffer_view->reference.count);
 836    } else {
 837       VkBufferView view;
 838       VkResult result = VKSCR(CreateBufferView)(screen->dev, bvci, NULL, &view);
 839       if (result != VK_SUCCESS) {
 840          mesa_loge("ZINK: vkCreateBufferView failed (%s)", vk_Result_to_str(result));
 841          goto out;
 842       }
 843       buffer_view = CALLOC_STRUCT(zink_buffer_view);
 844       if (!buffer_view) {
 845          VKSCR(DestroyBufferView)(screen->dev, view, NULL);
 846          goto out;
 847       }
 848       pipe_reference_init(&buffer_view->reference, 1);
 849       pipe_resource_reference(&buffer_view->pres, &res->base.b);
 850       buffer_view->bvci = *bvci;
 851       buffer_view->buffer_view = view;
 852       buffer_view->hash = hash;
 853       _mesa_hash_table_insert_pre_hashed(&res->bufferview_cache, hash, &buffer_view->bvci, buffer_view);
 854    }
 855 out:
 856    simple_mtx_unlock(&res->bufferview_mtx);
 857    return buffer_view;
 858 }
 859
 860 enum pipe_swizzle
 861 zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle)
 862 {
 863    switch (swizzle) {
 864    case PIPE_SWIZZLE_X:
 865    case PIPE_SWIZZLE_Y:
 866    case PIPE_SWIZZLE_Z:
 867    case PIPE_SWIZZLE_W:
 868       return desc->channel[swizzle].type == UTIL_FORMAT_TYPE_VOID ? PIPE_SWIZZLE_1 : swizzle;
 869    default:
 870       break;
 871    }
 872    return swizzle;
 873 }
 874
 875 ALWAYS_INLINE static enum pipe_swizzle
 876 clamp_zs_swizzle(enum pipe_swizzle swizzle)
 877 {
 878    switch (swizzle) {
 879    case PIPE_SWIZZLE_X:
 880    case PIPE_SWIZZLE_Y:
 881    case PIPE_SWIZZLE_Z:
 882    case PIPE_SWIZZLE_W:
 883       return PIPE_SWIZZLE_X;
 884    default:
 885       break;
 886    }
 887    return swizzle;
 888 }
 889
 890 ALWAYS_INLINE static enum pipe_swizzle
 891 clamp_alpha_swizzle(enum pipe_swizzle swizzle)
 892 {
 893    if (swizzle == PIPE_SWIZZLE_W)
 894       return PIPE_SWIZZLE_X;
 895    if (swizzle < PIPE_SWIZZLE_W)
 896       return PIPE_SWIZZLE_0;
 897    return swizzle;
 898 }
 899
 900 ALWAYS_INLINE static enum pipe_swizzle
 901 clamp_luminance_swizzle(enum pipe_swizzle swizzle)
 902 {
 903    if (swizzle == PIPE_SWIZZLE_W)
 904       return PIPE_SWIZZLE_1;
 905    if (swizzle < PIPE_SWIZZLE_W)
 906       return PIPE_SWIZZLE_X;
 907    return swizzle;
 908 }
 909
 910 ALWAYS_INLINE static enum pipe_swizzle
 911 clamp_luminance_alpha_swizzle(enum pipe_swizzle swizzle)
 912 {
 913    if (swizzle == PIPE_SWIZZLE_W)
 914       return PIPE_SWIZZLE_Y;
 915    if (swizzle < PIPE_SWIZZLE_W)
 916       return PIPE_SWIZZLE_X;
 917    return swizzle;
 918 }
 919
 920 ALWAYS_INLINE static bool
 921 viewtype_is_cube(const VkImageViewCreateInfo *ivci)
 922 {
 923    return ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE ||
 924           ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY;
 925 }
 926
 927 static struct pipe_sampler_view *
 928 zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres,
 929                          const struct pipe_sampler_view *state)
 930 {
 931    struct zink_screen *screen = zink_screen(pctx->screen);
 932    struct zink_resource *res = zink_resource(pres);
 933    struct zink_context *ctx = zink_context(pctx);
 934    struct zink_sampler_view *sampler_view = CALLOC_STRUCT_CL(zink_sampler_view);
 935    bool err;
 936
 937    sampler_view->base = *state;
 938    sampler_view->base.texture = NULL;
 939    pipe_resource_reference(&sampler_view->base.texture, pres);
 940    sampler_view->base.reference.count = 1;
 941    sampler_view->base.context = pctx;
 942
 943    if (state->target != PIPE_BUFFER) {
 944       VkImageViewCreateInfo ivci;
 945
 946       struct pipe_surface templ = {0};
 947       templ.u.tex.level = state->u.tex.first_level;
 948       templ.format = state->format;
 949       /* avoid needing mutable for depth/stencil sampling */
 950       if (util_format_is_depth_and_stencil(pres->format))
 951          templ.format = pres->format;
 952       if (state->target != PIPE_TEXTURE_3D) {
 953          templ.u.tex.first_layer = state->u.tex.first_layer;
 954          templ.u.tex.last_layer = state->u.tex.last_layer;
 955       }
 956
 957       if (zink_is_swapchain(res)) {
 958          if (!zink_kopper_acquire(ctx, res, UINT64_MAX)) {
 959             FREE_CL(sampler_view);
 960             return NULL;
 961          }
 962       }
 963
 964       ivci = create_ivci(screen, res, &templ, state->target);
 965       ivci.subresourceRange.levelCount = state->u.tex.last_level - state->u.tex.first_level + 1;
 966       ivci.subresourceRange.aspectMask = sampler_aspect_from_format(state->format);
 967       /* samplers for stencil aspects of packed formats need to always use stencil swizzle */
 968       if (ivci.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
 969          if (sampler_view->base.swizzle_r == PIPE_SWIZZLE_0 &&
 970              sampler_view->base.swizzle_g == PIPE_SWIZZLE_0 &&
 971              sampler_view->base.swizzle_b == PIPE_SWIZZLE_0 &&
 972              sampler_view->base.swizzle_a == PIPE_SWIZZLE_X) {
 973             /*
 974              * When the state tracker asks for 000x swizzles, this is depth mode GL_ALPHA,
 975              * however with the single dref fetch this will fail, so just spam all the channels.
 976              */
 977             ivci.components.r = VK_COMPONENT_SWIZZLE_R;
 978             ivci.components.g = VK_COMPONENT_SWIZZLE_R;
 979             ivci.components.b = VK_COMPONENT_SWIZZLE_R;
 980             ivci.components.a = VK_COMPONENT_SWIZZLE_R;
 981          } else {
 982             ivci.components.r = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_r));
 983             ivci.components.g = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_g));
 984             ivci.components.b = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_b));
 985             ivci.components.a = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_a));
 986          }
 987       } else {
 988          enum pipe_swizzle swizzle[4] = {
 989             sampler_view->base.swizzle_r,
 990             sampler_view->base.swizzle_g,
 991             sampler_view->base.swizzle_b,
 992             sampler_view->base.swizzle_a
 993          };
 994          /* if we have e.g., R8G8B8X8, then we have to ignore alpha since we're just emulating
 995           * these formats
 996           */
 997          if (zink_format_is_voidable_rgba_variant(state->format)) {
 998             const struct util_format_description *view_desc = util_format_description(state->format);
 999             for (int i = 0; i < 4; ++i)
1000                swizzle[i] = zink_clamp_void_swizzle(view_desc, swizzle[i]);
1001          } else if (util_format_is_alpha(state->format)) {
1002             for (int i = 0; i < 4; ++i)
1003                swizzle[i] = clamp_alpha_swizzle(swizzle[i]);
1004          } else if (util_format_is_luminance(pres->format) ||
1005                     util_format_is_luminance_alpha(pres->format)) {
1006             if (util_format_is_luminance(pres->format)) {
1007                for (int i = 0; i < 4; ++i)
1008                   swizzle[i] = clamp_luminance_swizzle(swizzle[i]);
1009             } else {
1010                for (int i = 0; i < 4; ++i)
1011                   swizzle[i] = clamp_luminance_alpha_swizzle(swizzle[i]);
1012             }
1013             if (state->format != pres->format) {
1014                /* luminance / luminance-alpha formats can be reinterpreted
1015                 * as red / red-alpha formats by the state-tracker, and we
1016                 * need to whack the green/blue channels here to the
1017                 * correct values for that to work.
1018                 */
1019                enum pipe_format linear = util_format_linear(pres->format);
1020                if (state->format == util_format_luminance_to_red(linear)) {
1021                   assert(swizzle[1] == PIPE_SWIZZLE_X ||
1022                          swizzle[1] == PIPE_SWIZZLE_0);
1023                   assert(swizzle[2] == PIPE_SWIZZLE_X ||
1024                          swizzle[2] == PIPE_SWIZZLE_0);
1025                   swizzle[1] = swizzle[2] = PIPE_SWIZZLE_0;
1026                } else
1027                   assert(state->format == linear);
1028             }
1029          }
1030
1031          ivci.components.r = zink_component_mapping(swizzle[0]);
1032          ivci.components.g = zink_component_mapping(swizzle[1]);
1033          ivci.components.b = zink_component_mapping(swizzle[2]);
1034          ivci.components.a = zink_component_mapping(swizzle[3]);
1035       }
1036       assert(ivci.format);
1037
1038       sampler_view->image_view = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci);
1039       if (!screen->info.have_EXT_non_seamless_cube_map && viewtype_is_cube(&sampler_view->image_view->ivci)) {
1040          ivci.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
1041          sampler_view->cube_array = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci);
1042       }
1043       err = !sampler_view->image_view;
1044    } else {
1045       VkBufferViewCreateInfo bvci = create_bvci(ctx, res, state->format, state->u.buf.offset, state->u.buf.size);
1046       sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
1047       err = !sampler_view->buffer_view;
1048    }
1049    if (err) {
1050       FREE_CL(sampler_view);
1051       return NULL;
1052    }
1053    return &sampler_view->base;
1054 }
1055
1056 void
1057 zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view)
1058 {
1059    struct zink_resource *res = zink_resource(buffer_view->pres);
1060    simple_mtx_lock(&res->bufferview_mtx);
1061    if (buffer_view->reference.count) {
1062       /* got a cache hit during deletion */
1063       simple_mtx_unlock(&res->bufferview_mtx);
1064       return;
1065    }
1066    struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, buffer_view->hash, &buffer_view->bvci);
1067    assert(he);
1068    _mesa_hash_table_remove(&res->bufferview_cache, he);
1069    simple_mtx_unlock(&res->bufferview_mtx);
1070    simple_mtx_lock(&res->obj->view_lock);
1071    util_dynarray_append(&res->obj->views, VkBufferView, buffer_view->buffer_view);
1072    simple_mtx_unlock(&res->obj->view_lock);
1073    pipe_resource_reference(&buffer_view->pres, NULL);
1074    FREE(buffer_view);
1075 }
1076
1077 static void
1078 zink_sampler_view_destroy(struct pipe_context *pctx,
1079                           struct pipe_sampler_view *pview)
1080 {
1081    struct zink_sampler_view *view = zink_sampler_view(pview);
1082    if (pview->texture->target == PIPE_BUFFER)
1083       zink_buffer_view_reference(zink_screen(pctx->screen), &view->buffer_view, NULL);
1084    else {
1085       zink_surface_reference(zink_screen(pctx->screen), &view->image_view, NULL);
1086       zink_surface_reference(zink_screen(pctx->screen), &view->cube_array, NULL);
1087    }
1088    pipe_resource_reference(&pview->texture, NULL);
1089    FREE_CL(view);
1090 }
1091
1092 static void
1093 zink_get_sample_position(struct pipe_context *ctx,
1094                          unsigned sample_count,
1095                          unsigned sample_index,
1096                          float *out_value)
1097 {
1098    /* TODO: handle this I guess */
1099    assert(zink_screen(ctx->screen)->info.props.limits.standardSampleLocations);
1100    /* from 26.4. Multisampling */
1101    switch (sample_count) {
1102    case 0:
1103    case 1: {
1104       float pos[][2] = { {0.5,0.5}, };
1105       out_value[0] = pos[sample_index][0];
1106       out_value[1] = pos[sample_index][1];
1107       break;
1108    }
1109    case 2: {
1110       float pos[][2] = { {0.75,0.75},
1111                         {0.25,0.25}, };
1112       out_value[0] = pos[sample_index][0];
1113       out_value[1] = pos[sample_index][1];
1114       break;
1115    }
1116    case 4: {
1117       float pos[][2] = { {0.375, 0.125},
1118                         {0.875, 0.375},
1119                         {0.125, 0.625},
1120                         {0.625, 0.875}, };
1121       out_value[0] = pos[sample_index][0];
1122       out_value[1] = pos[sample_index][1];
1123       break;
1124    }
1125    case 8: {
1126       float pos[][2] = { {0.5625, 0.3125},
1127                         {0.4375, 0.6875},
1128                         {0.8125, 0.5625},
1129                         {0.3125, 0.1875},
1130                         {0.1875, 0.8125},
1131                         {0.0625, 0.4375},
1132                         {0.6875, 0.9375},
1133                         {0.9375, 0.0625}, };
1134       out_value[0] = pos[sample_index][0];
1135       out_value[1] = pos[sample_index][1];
1136       break;
1137    }
1138    case 16: {
1139       float pos[][2] = { {0.5625, 0.5625},
1140                         {0.4375, 0.3125},
1141                         {0.3125, 0.625},
1142                         {0.75, 0.4375},
1143                         {0.1875, 0.375},
1144                         {0.625, 0.8125},
1145                         {0.8125, 0.6875},
1146                         {0.6875, 0.1875},
1147                         {0.375, 0.875},
1148                         {0.5, 0.0625},
1149                         {0.25, 0.125},
1150                         {0.125, 0.75},
1151                         {0.0, 0.5},
1152                         {0.9375, 0.25},
1153                         {0.875, 0.9375},
1154                         {0.0625, 0.0}, };
1155       out_value[0] = pos[sample_index][0];
1156       out_value[1] = pos[sample_index][1];
1157       break;
1158    }
1159    default:
1160       unreachable("unhandled sample count!");
1161    }
1162 }
1163
1164 static void
1165 zink_set_polygon_stipple(struct pipe_context *pctx,
1166                          const struct pipe_poly_stipple *ps)
1167 {
1168 }
1169
1170 ALWAYS_INLINE static void
1171 update_res_bind_count(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool decrement)
1172 {
1173    if (decrement) {
1174       assert(res->bind_count[is_compute]);
1175       if (!--res->bind_count[is_compute])
1176          _mesa_set_remove_key(ctx->need_barriers[is_compute], res);
1177       check_resource_for_batch_ref(ctx, res);
1178    } else
1179       res->bind_count[is_compute]++;
1180 }
1181
1182 ALWAYS_INLINE static void
1183 update_existing_vbo(struct zink_context *ctx, unsigned slot)
1184 {
1185    if (!ctx->vertex_buffers[slot].buffer.resource)
1186       return;
1187    struct zink_resource *res = zink_resource(ctx->vertex_buffers[slot].buffer.resource);
1188    res->vbo_bind_count--;
1189    res->vbo_bind_mask &= ~BITFIELD_BIT(slot);
1190    if (!res->vbo_bind_count) {
1191       res->gfx_barrier &= ~VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1192       res->barrier_access[0] &= ~VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1193    }
1194    update_res_bind_count(ctx, res, false, true);
1195 }
1196
1197 static void
1198 zink_set_vertex_buffers(struct pipe_context *pctx,
1199                         unsigned start_slot,
1200                         unsigned num_buffers,
1201                         unsigned unbind_num_trailing_slots,
1202                         bool take_ownership,
1203                         const struct pipe_vertex_buffer *buffers)
1204 {
1205    struct zink_context *ctx = zink_context(pctx);
1206    const bool have_input_state = zink_screen(pctx->screen)->info.have_EXT_vertex_input_dynamic_state;
1207    const bool need_state_change = !zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state &&
1208                                   !have_input_state;
1209    uint32_t enabled_buffers = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
1210    enabled_buffers |= u_bit_consecutive(start_slot, num_buffers);
1211    enabled_buffers &= ~u_bit_consecutive(start_slot + num_buffers, unbind_num_trailing_slots);
1212    bool stride_changed = false;
1213
1214    if (buffers) {
1215       for (unsigned i = 0; i < num_buffers; ++i) {
1216          const struct pipe_vertex_buffer *vb = buffers + i;
1217          struct pipe_vertex_buffer *ctx_vb = &ctx->vertex_buffers[start_slot + i];
1218          stride_changed |= ctx_vb->stride != vb->stride;
1219          update_existing_vbo(ctx, start_slot + i);
1220          if (!take_ownership)
1221             pipe_resource_reference(&ctx_vb->buffer.resource, vb->buffer.resource);
1222          else {
1223             pipe_resource_reference(&ctx_vb->buffer.resource, NULL);
1224             ctx_vb->buffer.resource = vb->buffer.resource;
1225          }
1226          if (vb->buffer.resource) {
1227             struct zink_resource *res = zink_resource(vb->buffer.resource);
1228             res->vbo_bind_mask |= BITFIELD_BIT(start_slot + i);
1229             res->vbo_bind_count++;
1230             res->gfx_barrier |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1231             res->barrier_access[0] |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1232             update_res_bind_count(ctx, res, false, false);
1233             ctx_vb->stride = vb->stride;
1234             ctx_vb->buffer_offset = vb->buffer_offset;
1235             zink_batch_resource_usage_set(&ctx->batch, res, false, true);
1236             /* always barrier before possible rebind */
1237             zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
1238                                          VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
1239             res->obj->unordered_read = false;
1240          } else {
1241             enabled_buffers &= ~BITFIELD_BIT(start_slot + i);
1242          }
1243       }
1244    } else {
1245       for (unsigned i = 0; i < num_buffers; ++i) {
1246          update_existing_vbo(ctx, start_slot + i);
1247          pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL);
1248       }
1249    }
1250    for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
1251       update_existing_vbo(ctx, start_slot + i);
1252       pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL);
1253    }
1254    if (need_state_change)
1255       ctx->vertex_state_changed = true;
1256    else if (!have_input_state && (stride_changed || ctx->gfx_pipeline_state.vertex_buffers_enabled_mask != enabled_buffers))
1257       ctx->vertex_state_changed = true;
1258    ctx->gfx_pipeline_state.vertex_buffers_enabled_mask = enabled_buffers;
1259    ctx->vertex_buffers_dirty = num_buffers > 0;
1260 #ifndef NDEBUG
1261    u_foreach_bit(b, enabled_buffers)
1262       assert(ctx->vertex_buffers[b].buffer.resource);
1263 #endif
1264 }
1265
1266 static void
1267 zink_set_viewport_states(struct pipe_context *pctx,
1268                          unsigned start_slot,
1269                          unsigned num_viewports,
1270                          const struct pipe_viewport_state *state)
1271 {
1272    struct zink_context *ctx = zink_context(pctx);
1273
1274    for (unsigned i = 0; i < num_viewports; ++i)
1275       ctx->vp_state.viewport_states[start_slot + i] = state[i];
1276
1277    ctx->vp_state_changed = true;
1278 }
1279
1280 static void
1281 zink_set_scissor_states(struct pipe_context *pctx,
1282                         unsigned start_slot, unsigned num_scissors,
1283                         const struct pipe_scissor_state *states)
1284 {
1285    struct zink_context *ctx = zink_context(pctx);
1286
1287    for (unsigned i = 0; i < num_scissors; i++)
1288       ctx->vp_state.scissor_states[start_slot + i] = states[i];
1289    ctx->scissor_changed = true;
1290 }
1291
1292 static void
1293 zink_set_inlinable_constants(struct pipe_context *pctx,
1294                              gl_shader_stage shader,
1295                              uint num_values, uint32_t *values)
1296 {
1297    struct zink_context *ctx = (struct zink_context *)pctx;
1298    const uint32_t bit = BITFIELD_BIT(shader);
1299    uint32_t *inlinable_uniforms;
1300    struct zink_shader_key *key = NULL;
1301
1302    if (shader == MESA_SHADER_COMPUTE) {
1303       key = &ctx->compute_pipeline_state.key;
1304    } else {
1305       assert(!zink_screen(pctx->screen)->optimal_keys);
1306       key = &ctx->gfx_pipeline_state.shader_keys.key[shader];
1307    }
1308    inlinable_uniforms = key->base.inlined_uniform_values;
1309    if (!(ctx->inlinable_uniforms_valid_mask & bit) ||
1310        memcmp(inlinable_uniforms, values, num_values * 4)) {
1311       memcpy(inlinable_uniforms, values, num_values * 4);
1312       if (shader == MESA_SHADER_COMPUTE)
1313          ctx->compute_dirty = true;
1314       else
1315          ctx->dirty_gfx_stages |= bit;
1316       ctx->inlinable_uniforms_valid_mask |= bit;
1317       key->inline_uniforms = true;
1318    }
1319 }
1320
1321 ALWAYS_INLINE static void
1322 unbind_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1323 {
1324    if (!res->sampler_binds[pstage] && !res->image_binds[pstage])
1325       res->gfx_barrier &= ~zink_pipeline_flags_from_pipe_stage(pstage);
1326 }
1327
1328 ALWAYS_INLINE static void
1329 unbind_buffer_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1330 {
1331    if (!res->ubo_bind_mask[pstage] && !res->ssbo_bind_mask[pstage])
1332       unbind_descriptor_stage(res, pstage);
1333 }
1334
1335 ALWAYS_INLINE static void
1336 unbind_ubo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot)
1337 {
1338    if (!res)
1339       return;
1340    res->ubo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1341    res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1342    unbind_buffer_descriptor_stage(res, pstage);
1343    if (!res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE])
1344       res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_UNIFORM_READ_BIT;
1345    update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1346 }
1347
1348 static void
1349 invalidate_inlined_uniforms(struct zink_context *ctx, gl_shader_stage pstage)
1350 {
1351    unsigned bit = BITFIELD_BIT(pstage);
1352    if (!(ctx->inlinable_uniforms_valid_mask & bit))
1353       return;
1354    ctx->inlinable_uniforms_valid_mask &= ~bit;
1355    if (pstage == MESA_SHADER_COMPUTE) {
1356       ctx->compute_dirty = true;
1357       return;
1358    }
1359    assert(!zink_screen(ctx->base.screen)->optimal_keys);
1360    ctx->dirty_gfx_stages |= bit;
1361    struct zink_shader_key *key = &ctx->gfx_pipeline_state.shader_keys.key[pstage];
1362    key->inline_uniforms = false;
1363 }
1364
1365 static void
1366 zink_set_constant_buffer(struct pipe_context *pctx,
1367                          gl_shader_stage shader, uint index,
1368                          bool take_ownership,
1369                          const struct pipe_constant_buffer *cb)
1370 {
1371    struct zink_context *ctx = zink_context(pctx);
1372    bool update = false;
1373
1374    struct zink_resource *res = zink_resource(ctx->ubos[shader][index].buffer);
1375    if (cb) {
1376       struct pipe_resource *buffer = cb->buffer;
1377       unsigned offset = cb->buffer_offset;
1378       struct zink_screen *screen = zink_screen(pctx->screen);
1379       if (cb->user_buffer) {
1380          u_upload_data(ctx->base.const_uploader, 0, cb->buffer_size,
1381                        screen->info.props.limits.minUniformBufferOffsetAlignment,
1382                        cb->user_buffer, &offset, &buffer);
1383       }
1384       struct zink_resource *new_res = zink_resource(buffer);
1385       if (new_res) {
1386          if (new_res != res) {
1387             unbind_ubo(ctx, res, shader, index);
1388             new_res->ubo_bind_count[shader == MESA_SHADER_COMPUTE]++;
1389             new_res->ubo_bind_mask[shader] |= BITFIELD_BIT(index);
1390             new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader);
1391             new_res->barrier_access[shader == MESA_SHADER_COMPUTE] |= VK_ACCESS_UNIFORM_READ_BIT;
1392             update_res_bind_count(ctx, new_res, shader == MESA_SHADER_COMPUTE, false);
1393          }
1394          zink_batch_resource_usage_set(&ctx->batch, new_res, false, true);
1395          zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, VK_ACCESS_UNIFORM_READ_BIT,
1396                                       new_res->gfx_barrier);
1397          new_res->obj->unordered_read = false;
1398       }
1399       update |= ctx->ubos[shader][index].buffer_offset != offset ||
1400                 !!res != !!buffer || (res && res->obj->buffer != new_res->obj->buffer) ||
1401                 ctx->ubos[shader][index].buffer_size != cb->buffer_size;
1402
1403       if (take_ownership) {
1404          pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1405          ctx->ubos[shader][index].buffer = buffer;
1406       } else {
1407          pipe_resource_reference(&ctx->ubos[shader][index].buffer, buffer);
1408       }
1409       ctx->ubos[shader][index].buffer_offset = offset;
1410       ctx->ubos[shader][index].buffer_size = cb->buffer_size;
1411       ctx->ubos[shader][index].user_buffer = NULL;
1412
1413       if (cb->user_buffer)
1414          pipe_resource_reference(&buffer, NULL);
1415
1416       if (index + 1 >= ctx->di.num_ubos[shader])
1417          ctx->di.num_ubos[shader] = index + 1;
1418       update_descriptor_state_ubo(ctx, shader, index, new_res);
1419    } else {
1420       ctx->ubos[shader][index].buffer_offset = 0;
1421       ctx->ubos[shader][index].buffer_size = 0;
1422       ctx->ubos[shader][index].user_buffer = NULL;
1423       if (res) {
1424          unbind_ubo(ctx, res, shader, index);
1425          update_descriptor_state_ubo(ctx, shader, index, NULL);
1426       }
1427       update = !!ctx->ubos[shader][index].buffer;
1428
1429       pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1430       if (ctx->di.num_ubos[shader] == index + 1)
1431          ctx->di.num_ubos[shader]--;
1432    }
1433    if (index == 0) {
1434       /* Invalidate current inlinable uniforms. */
1435       invalidate_inlined_uniforms(ctx, shader);
1436    }
1437
1438    if (update)
1439       zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, index, 1);
1440 }
1441
1442 ALWAYS_INLINE static void
1443 unbind_descriptor_reads(struct zink_resource *res, gl_shader_stage pstage)
1444 {
1445    if (!res->sampler_binds[pstage] && !res->image_binds[pstage])
1446       res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_READ_BIT;
1447 }
1448
1449 ALWAYS_INLINE static void
1450 unbind_buffer_descriptor_reads(struct zink_resource *res, gl_shader_stage pstage)
1451 {
1452    if (!res->ssbo_bind_count[pstage == MESA_SHADER_COMPUTE])
1453       unbind_descriptor_reads(res, pstage);
1454 }
1455
1456 ALWAYS_INLINE static void
1457 unbind_ssbo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot, bool writable)
1458 {
1459    if (!res)
1460       return;
1461    res->ssbo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1462    res->ssbo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1463    unbind_buffer_descriptor_stage(res, pstage);
1464    unbind_buffer_descriptor_reads(res, pstage);
1465    update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1466    if (writable)
1467       res->write_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1468    if (!res->write_bind_count[pstage == MESA_SHADER_COMPUTE])
1469       res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1470 }
1471
1472 static void
1473 zink_set_shader_buffers(struct pipe_context *pctx,
1474                         gl_shader_stage p_stage,
1475                         unsigned start_slot, unsigned count,
1476                         const struct pipe_shader_buffer *buffers,
1477                         unsigned writable_bitmask)
1478 {
1479    struct zink_context *ctx = zink_context(pctx);
1480    bool update = false;
1481    unsigned max_slot = 0;
1482
1483    unsigned modified_bits = u_bit_consecutive(start_slot, count);
1484    unsigned old_writable_mask = ctx->writable_ssbos[p_stage];
1485    ctx->writable_ssbos[p_stage] &= ~modified_bits;
1486    ctx->writable_ssbos[p_stage] |= writable_bitmask << start_slot;
1487
1488    for (unsigned i = 0; i < count; i++) {
1489       struct pipe_shader_buffer *ssbo = &ctx->ssbos[p_stage][start_slot + i];
1490       struct zink_resource *res = ssbo->buffer ? zink_resource(ssbo->buffer) : NULL;
1491       bool was_writable = old_writable_mask & BITFIELD64_BIT(start_slot + i);
1492       if (buffers && buffers[i].buffer) {
1493          struct zink_resource *new_res = zink_resource(buffers[i].buffer);
1494          if (new_res != res) {
1495             unbind_ssbo(ctx, res, p_stage, i, was_writable);
1496             new_res->ssbo_bind_mask[p_stage] |= BITFIELD_BIT(i);
1497             new_res->ssbo_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1498             new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage);
1499             update_res_bind_count(ctx, new_res, p_stage == MESA_SHADER_COMPUTE, false);
1500          }
1501          VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT;
1502          if (ctx->writable_ssbos[p_stage] & BITFIELD64_BIT(start_slot + i)) {
1503             new_res->write_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1504             access |= VK_ACCESS_SHADER_WRITE_BIT;
1505          }
1506          pipe_resource_reference(&ssbo->buffer, &new_res->base.b);
1507          new_res->barrier_access[p_stage == MESA_SHADER_COMPUTE] |= access;
1508          zink_batch_resource_usage_set(&ctx->batch, new_res, access & VK_ACCESS_SHADER_WRITE_BIT, true);
1509          ssbo->buffer_offset = buffers[i].buffer_offset;
1510          ssbo->buffer_size = MIN2(buffers[i].buffer_size, new_res->base.b.width0 - ssbo->buffer_offset);
1511          util_range_add(&new_res->base.b, &new_res->valid_buffer_range, ssbo->buffer_offset,
1512                         ssbo->buffer_offset + ssbo->buffer_size);
1513          zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, access,
1514                                       new_res->gfx_barrier);
1515          update = true;
1516          max_slot = MAX2(max_slot, start_slot + i);
1517          update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, new_res);
1518          if (zink_resource_access_is_write(access))
1519             new_res->obj->unordered_read = new_res->obj->unordered_write = false;
1520          else
1521             new_res->obj->unordered_read = false;
1522       } else {
1523          update = !!res;
1524          ssbo->buffer_offset = 0;
1525          ssbo->buffer_size = 0;
1526          if (res) {
1527             unbind_ssbo(ctx, res, p_stage, i, was_writable);
1528             update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, NULL);
1529          }
1530          pipe_resource_reference(&ssbo->buffer, NULL);
1531       }
1532    }
1533    if (start_slot + count >= ctx->di.num_ssbos[p_stage])
1534       ctx->di.num_ssbos[p_stage] = max_slot + 1;
1535    if (update)
1536       zink_context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_SSBO, start_slot, count);
1537 }
1538
1539 static void
1540 update_binds_for_samplerviews(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1541 {
1542     VkImageLayout layout = get_layout_for_binding(ctx, res, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, is_compute);
1543     if (is_compute) {
1544        u_foreach_bit(slot, res->sampler_binds[MESA_SHADER_COMPUTE]) {
1545           if (ctx->di.textures[MESA_SHADER_COMPUTE][slot].imageLayout != layout) {
1546              update_descriptor_state_sampler(ctx, MESA_SHADER_COMPUTE, slot, res);
1547              zink_context_invalidate_descriptor_state(ctx, MESA_SHADER_COMPUTE, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1548           }
1549        }
1550     } else {
1551        for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++) {
1552           u_foreach_bit(slot, res->sampler_binds[i]) {
1553              if (ctx->di.textures[i][slot].imageLayout != layout) {
1554                 update_descriptor_state_sampler(ctx, i, slot, res);
1555                 zink_context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1556              }
1557           }
1558        }
1559     }
1560 }
1561
1562 static void
1563 flush_pending_clears(struct zink_context *ctx, struct zink_resource *res)
1564 {
1565    if (res->fb_bind_count && ctx->clears_enabled)
1566       zink_fb_clears_apply(ctx, &res->base.b);
1567 }
1568
1569 static inline void
1570 unbind_shader_image_counts(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool writable)
1571 {
1572    update_res_bind_count(ctx, res, is_compute, true);
1573    if (writable)
1574       res->write_bind_count[is_compute]--;
1575    res->image_bind_count[is_compute]--;
1576    /* if this was the last image bind, the sampler bind layouts must be updated */
1577    if (!res->obj->is_buffer && !res->image_bind_count[is_compute] && res->bind_count[is_compute])
1578       update_binds_for_samplerviews(ctx, res, is_compute);
1579 }
1580
1581 ALWAYS_INLINE static void
1582 check_for_layout_update(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1583 {
1584    VkImageLayout layout = res->bind_count[is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1585    VkImageLayout other_layout = res->bind_count[!is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, !is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1586    if (res->bind_count[is_compute] && layout && res->layout != layout)
1587       _mesa_set_add(ctx->need_barriers[is_compute], res);
1588    if (res->bind_count[!is_compute] && other_layout && (layout != other_layout || res->layout != other_layout))
1589       _mesa_set_add(ctx->need_barriers[!is_compute], res);
1590 }
1591
1592 static void
1593 unbind_shader_image(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
1594 {
1595    struct zink_image_view *image_view = &ctx->image_views[stage][slot];
1596    bool is_compute = stage == MESA_SHADER_COMPUTE;
1597    if (!image_view->base.resource)
1598       return;
1599
1600    struct zink_resource *res = zink_resource(image_view->base.resource);
1601    res->image_binds[stage] &= ~BITFIELD_BIT(slot);
1602    unbind_shader_image_counts(ctx, res, is_compute, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE);
1603    if (!res->write_bind_count[is_compute])
1604       res->barrier_access[stage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1605
1606    if (image_view->base.resource->target == PIPE_BUFFER) {
1607       unbind_buffer_descriptor_stage(res, stage);
1608       unbind_buffer_descriptor_reads(res, stage);
1609       zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
1610    } else {
1611       unbind_descriptor_stage(res, stage);
1612       if (!res->image_bind_count[is_compute])
1613          check_for_layout_update(ctx, res, is_compute);
1614       zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
1615    }
1616    image_view->base.resource = NULL;
1617    image_view->surface = NULL;
1618 }
1619
1620 static struct zink_buffer_view *
1621 create_image_bufferview(struct zink_context *ctx, const struct pipe_image_view *view)
1622 {
1623    struct zink_resource *res = zink_resource(view->resource);
1624    VkBufferViewCreateInfo bvci = create_bvci(ctx, res, view->format, view->u.buf.offset, view->u.buf.size);
1625    struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1626    if (!buffer_view)
1627       return NULL;
1628    util_range_add(&res->base.b, &res->valid_buffer_range, view->u.buf.offset,
1629                   view->u.buf.offset + view->u.buf.size);
1630    return buffer_view;
1631 }
1632
1633 static void
1634 finalize_image_bind(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1635 {
1636    /* if this is the first image bind and there are sampler binds, the image's sampler layout
1637     * must be updated to GENERAL
1638     */
1639    if (res->image_bind_count[is_compute] == 1 &&
1640        res->bind_count[is_compute] > 1)
1641       update_binds_for_samplerviews(ctx, res, is_compute);
1642    check_for_layout_update(ctx, res, is_compute);
1643 }
1644
1645 static struct zink_surface *
1646 create_image_surface(struct zink_context *ctx, const struct pipe_image_view *view, bool is_compute)
1647 {
1648    struct zink_screen *screen = zink_screen(ctx->base.screen);
1649    struct zink_resource *res = zink_resource(view->resource);
1650    struct pipe_surface tmpl = {0};
1651    enum pipe_texture_target target = res->base.b.target;
1652    tmpl.format = view->format;
1653    tmpl.u.tex.level = view->u.tex.level;
1654    tmpl.u.tex.first_layer = view->u.tex.first_layer;
1655    tmpl.u.tex.last_layer = view->u.tex.last_layer;
1656    unsigned depth = 1 + tmpl.u.tex.last_layer - tmpl.u.tex.first_layer;
1657    switch (target) {
1658    case PIPE_TEXTURE_3D:
1659       if (depth < u_minify(res->base.b.depth0, view->u.tex.level)) {
1660          assert(depth == 1);
1661          target = PIPE_TEXTURE_2D;
1662          if (!screen->info.have_EXT_image_2d_view_of_3d ||
1663              !screen->info.view2d_feats.image2DViewOf3D) {
1664             static bool warned = false;
1665             warn_missing_feature(warned, "image2DViewOf3D");
1666          }
1667       } else {
1668          assert(tmpl.u.tex.first_layer == 0);
1669          tmpl.u.tex.last_layer = 0;
1670       }
1671       break;
1672    case PIPE_TEXTURE_2D_ARRAY:
1673    case PIPE_TEXTURE_1D_ARRAY:
1674       if (depth < res->base.b.array_size && depth == 1)
1675          target = target == PIPE_TEXTURE_2D_ARRAY ? PIPE_TEXTURE_2D : PIPE_TEXTURE_1D;
1676       break;
1677    default: break;
1678    }
1679    if (!res->obj->dt && view->resource->format != view->format)
1680       /* mutable not set by default */
1681       zink_resource_object_init_mutable(ctx, res);
1682    VkImageViewCreateInfo ivci = create_ivci(screen, res, &tmpl, target);
1683    struct pipe_surface *psurf = zink_get_surface(ctx, view->resource, &tmpl, &ivci);
1684    if (!psurf)
1685       return NULL;
1686    struct zink_surface *surface = zink_surface(psurf);
1687    if (is_compute)
1688       flush_pending_clears(ctx, res);
1689    return surface;
1690 }
1691
1692 static void
1693 zink_set_shader_images(struct pipe_context *pctx,
1694                        gl_shader_stage p_stage,
1695                        unsigned start_slot, unsigned count,
1696                        unsigned unbind_num_trailing_slots,
1697                        const struct pipe_image_view *images)
1698 {
1699    struct zink_context *ctx = zink_context(pctx);
1700    bool update = false;
1701    for (unsigned i = 0; i < count; i++) {
1702       struct zink_image_view *image_view = &ctx->image_views[p_stage][start_slot + i];
1703       if (images && images[i].resource) {
1704          struct zink_resource *res = zink_resource(images[i].resource);
1705          if (!zink_resource_object_init_storage(ctx, res)) {
1706             debug_printf("couldn't create storage image!");
1707             continue;
1708          }
1709          /* no refs */
1710          VkAccessFlags access = 0;
1711          if (images[i].access & PIPE_IMAGE_ACCESS_WRITE) {
1712             res->write_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1713             access |= VK_ACCESS_SHADER_WRITE_BIT;
1714          }
1715          if (images[i].access & PIPE_IMAGE_ACCESS_READ) {
1716             access |= VK_ACCESS_SHADER_READ_BIT;
1717          }
1718          res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage);
1719          res->barrier_access[p_stage == MESA_SHADER_COMPUTE] |= access;
1720          if (images[i].resource->target == PIPE_BUFFER) {
1721             struct zink_buffer_view *bv = create_image_bufferview(ctx, &images[i]);
1722             assert(bv);
1723             if (image_view->buffer_view != bv) {
1724                update_res_bind_count(ctx, res, p_stage == MESA_SHADER_COMPUTE, false);
1725                res->image_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1726                unbind_shader_image(ctx, p_stage, start_slot + i);
1727             }
1728             image_view->buffer_view = bv;
1729             zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access,
1730                                          res->gfx_barrier);
1731             zink_batch_resource_usage_set(&ctx->batch, res,
1732                                           zink_resource_access_is_write(access), true);
1733          } else {
1734             struct zink_surface *surface = create_image_surface(ctx, &images[i], p_stage == MESA_SHADER_COMPUTE);
1735             assert(surface);
1736             if (image_view->surface != surface) {
1737                res->image_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1738                update_res_bind_count(ctx, res, p_stage == MESA_SHADER_COMPUTE, false);
1739                unbind_shader_image(ctx, p_stage, start_slot + i);
1740                image_view->surface = surface;
1741             } else {
1742                /* create_image_surface will always increment ref */
1743                zink_surface_reference(zink_screen(ctx->base.screen), &surface, NULL);
1744             }
1745             finalize_image_bind(ctx, res, p_stage == MESA_SHADER_COMPUTE);
1746             zink_batch_resource_usage_set(&ctx->batch, res,
1747                                           zink_resource_access_is_write(access), false);
1748          }
1749          memcpy(&image_view->base, images + i, sizeof(struct pipe_image_view));
1750          update = true;
1751          update_descriptor_state_image(ctx, p_stage, start_slot + i, res);
1752          if (zink_resource_access_is_write(access) || !res->obj->is_buffer)
1753             res->obj->unordered_read = res->obj->unordered_write = false;
1754          else
1755             res->obj->unordered_read = false;
1756          res->image_binds[p_stage] |= BITFIELD_BIT(start_slot + i);
1757       } else if (image_view->base.resource) {
1758          update = true;
1759
1760          unbind_shader_image(ctx, p_stage, start_slot + i);
1761          update_descriptor_state_image(ctx, p_stage, start_slot + i, NULL);
1762       }
1763    }
1764    for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
1765       update |= !!ctx->image_views[p_stage][start_slot + count + i].base.resource;
1766       unbind_shader_image(ctx, p_stage, start_slot + count + i);
1767       update_descriptor_state_image(ctx, p_stage, start_slot + count + i, NULL);
1768    }
1769    ctx->di.num_images[p_stage] = start_slot + count;
1770    if (update)
1771       zink_context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_IMAGE, start_slot, count);
1772 }
1773
1774 ALWAYS_INLINE static void
1775 unbind_samplerview(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
1776 {
1777    struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][slot]);
1778    if (!sv || !sv->base.texture)
1779       return;
1780    struct zink_resource *res = zink_resource(sv->base.texture);
1781    res->sampler_bind_count[stage == MESA_SHADER_COMPUTE]--;
1782    if (stage != MESA_SHADER_COMPUTE && !res->sampler_bind_count[0] && res->fb_bind_count) {
1783       u_foreach_bit(idx, res->fb_binds) {
1784          if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
1785             ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
1786             ctx->rp_layout_changed = true;
1787          }
1788          unsigned feedback_loops = ctx->feedback_loops;
1789          ctx->feedback_loops &= ~BITFIELD_BIT(idx);
1790          if (feedback_loops != ctx->feedback_loops) {
1791             if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
1792                if (ctx->gfx_pipeline_state.feedback_loop_zs)
1793                   ctx->gfx_pipeline_state.dirty = true;
1794                ctx->gfx_pipeline_state.feedback_loop_zs = false;
1795             } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
1796                if (ctx->gfx_pipeline_state.feedback_loop)
1797                   ctx->gfx_pipeline_state.dirty = true;
1798                ctx->gfx_pipeline_state.feedback_loop = false;
1799             }
1800          }
1801       }
1802    }
1803    update_res_bind_count(ctx, res, stage == MESA_SHADER_COMPUTE, true);
1804    res->sampler_binds[stage] &= ~BITFIELD_BIT(slot);
1805    if (res->obj->is_buffer) {
1806       unbind_buffer_descriptor_stage(res, stage);
1807       unbind_buffer_descriptor_reads(res, stage);
1808    } else {
1809       unbind_descriptor_stage(res, stage);
1810       unbind_descriptor_reads(res, stage);
1811    }
1812 }
1813
1814 static void
1815 zink_set_sampler_views(struct pipe_context *pctx,
1816                        gl_shader_stage shader_type,
1817                        unsigned start_slot,
1818                        unsigned num_views,
1819                        unsigned unbind_num_trailing_slots,
1820                        bool take_ownership,
1821                        struct pipe_sampler_view **views)
1822 {
1823    struct zink_context *ctx = zink_context(pctx);
1824    unsigned i;
1825
1826    const uint32_t mask = BITFIELD_RANGE(start_slot, num_views);
1827    ctx->di.cubes[shader_type] &= ~mask;
1828
1829    bool update = false;
1830    for (i = 0; i < num_views; ++i) {
1831       struct pipe_sampler_view *pview = views ? views[i] : NULL;
1832       struct zink_sampler_view *a = zink_sampler_view(ctx->sampler_views[shader_type][start_slot + i]);
1833       struct zink_sampler_view *b = zink_sampler_view(pview);
1834       struct zink_resource *res = b ? zink_resource(b->base.texture) : NULL;
1835       if (b && b->base.texture) {
1836          if (!a || zink_resource(a->base.texture) != res) {
1837             if (a)
1838                unbind_samplerview(ctx, shader_type, start_slot + i);
1839             update_res_bind_count(ctx, res, shader_type == MESA_SHADER_COMPUTE, false);
1840             res->sampler_bind_count[shader_type == MESA_SHADER_COMPUTE]++;
1841             res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type);
1842             res->barrier_access[shader_type == MESA_SHADER_COMPUTE] |= VK_ACCESS_SHADER_READ_BIT;
1843          }
1844          if (res->base.b.target == PIPE_BUFFER) {
1845             if (b->buffer_view->bvci.buffer != res->obj->buffer) {
1846                /* if this resource has been rebound while it wasn't set here,
1847                 * its backing resource will have changed and thus we need to update
1848                 * the bufferview
1849                 */
1850                VkBufferViewCreateInfo bvci = b->buffer_view->bvci;
1851                bvci.buffer = res->obj->buffer;
1852                struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1853                assert(buffer_view != b->buffer_view);
1854                zink_buffer_view_reference(zink_screen(ctx->base.screen), &b->buffer_view, NULL);
1855                b->buffer_view = buffer_view;
1856                update = true;
1857             } else if (!a || a->buffer_view->buffer_view != b->buffer_view->buffer_view)
1858                   update = true;
1859             zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT,
1860                                          res->gfx_barrier);
1861             zink_batch_resource_usage_set(&ctx->batch, res, false, true);
1862          } else if (!res->obj->is_buffer) {
1863              if (res->base.b.format != b->image_view->base.format)
1864                 /* mutable not set by default */
1865                 zink_resource_object_init_mutable(ctx, res);
1866              if (res->obj != b->image_view->obj) {
1867                 struct pipe_surface *psurf = &b->image_view->base;
1868                 VkImageView iv = b->image_view->image_view;
1869                 zink_rebind_surface(ctx, &psurf);
1870                 b->image_view = zink_surface(psurf);
1871                 update |= iv != b->image_view->image_view;
1872              } else  if (a != b)
1873                 update = true;
1874              if (shader_type == MESA_SHADER_COMPUTE)
1875                 flush_pending_clears(ctx, res);
1876              if (b->cube_array) {
1877                 ctx->di.cubes[shader_type] |= BITFIELD_BIT(start_slot + i);
1878              }
1879              check_for_layout_update(ctx, res, shader_type == MESA_SHADER_COMPUTE);
1880              if (!a)
1881                 update = true;
1882              zink_batch_resource_usage_set(&ctx->batch, res, false, false);
1883              res->obj->unordered_write = false;
1884          }
1885          res->sampler_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
1886          res->obj->unordered_read = false;
1887       } else if (a) {
1888          unbind_samplerview(ctx, shader_type, start_slot + i);
1889          update = true;
1890       }
1891       if (take_ownership) {
1892          pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], NULL);
1893          ctx->sampler_views[shader_type][start_slot + i] = pview;
1894       } else {
1895          pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], pview);
1896       }
1897       update_descriptor_state_sampler(ctx, shader_type, start_slot + i, res);
1898    }
1899    for (; i < num_views + unbind_num_trailing_slots; ++i) {
1900       update |= !!ctx->sampler_views[shader_type][start_slot + i];
1901       unbind_samplerview(ctx, shader_type, start_slot + i);
1902       pipe_sampler_view_reference(
1903          &ctx->sampler_views[shader_type][start_slot + i],
1904          NULL);
1905       update_descriptor_state_sampler(ctx, shader_type, start_slot + i, NULL);
1906    }
1907    ctx->di.num_sampler_views[shader_type] = start_slot + num_views;
1908    if (update) {
1909       struct zink_screen *screen = zink_screen(pctx->screen);
1910       zink_context_invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, num_views);
1911       if (!screen->info.have_EXT_non_seamless_cube_map)
1912          update_nonseamless_shader_key(ctx, shader_type);
1913    }
1914 }
1915
1916 static uint64_t
1917 zink_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *state)
1918 {
1919    struct zink_context *ctx = zink_context(pctx);
1920    struct zink_resource *res = zink_resource(view->texture);
1921    struct zink_sampler_view *sv = zink_sampler_view(view);
1922    struct zink_bindless_descriptor *bd;
1923    bd = calloc(1, sizeof(struct zink_bindless_descriptor));
1924    if (!bd)
1925       return 0;
1926
1927    bd->sampler = pctx->create_sampler_state(pctx, state);
1928    if (!bd->sampler) {
1929       free(bd);
1930       return 0;
1931    }
1932
1933    bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
1934    if (res->base.b.target == PIPE_BUFFER)
1935       zink_buffer_view_reference(zink_screen(pctx->screen), &bd->ds.bufferview, sv->buffer_view);
1936    else
1937       zink_surface_reference(zink_screen(pctx->screen), &bd->ds.surface, sv->image_view);
1938    uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].tex_slots);
1939    if (bd->ds.is_buffer)
1940       handle += ZINK_MAX_BINDLESS_HANDLES;
1941    bd->handle = handle;
1942    _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].tex_handles, (void*)(uintptr_t)handle, bd);
1943    return handle;
1944 }
1945
1946 static void
1947 zink_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
1948 {
1949    struct zink_context *ctx = zink_context(pctx);
1950    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
1951    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
1952    assert(he);
1953    struct zink_bindless_descriptor *bd = he->data;
1954    struct zink_descriptor_surface *ds = &bd->ds;
1955    _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].tex_handles, he);
1956    uint32_t h = handle;
1957    util_dynarray_append(&ctx->batch.state->bindless_releases[0], uint32_t, h);
1958
1959    if (ds->is_buffer) {
1960       zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
1961    } else {
1962       zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
1963       pctx->delete_sampler_state(pctx, bd->sampler);
1964    }
1965    free(ds);
1966 }
1967
1968 static void
1969 rebind_bindless_bufferview(struct zink_context *ctx, struct zink_resource *res, struct zink_descriptor_surface *ds)
1970 {
1971    /* if this resource has been rebound while it wasn't set here,
1972     * its backing resource will have changed and thus we need to update
1973     * the bufferview
1974     */
1975    VkBufferViewCreateInfo bvci = ds->bufferview->bvci;
1976    bvci.buffer = res->obj->buffer;
1977    struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1978    assert(buffer_view != ds->bufferview);
1979    zink_buffer_view_reference(zink_screen(ctx->base.screen), &ds->bufferview, NULL);
1980    ds->bufferview = buffer_view;
1981 }
1982
1983 static void
1984 zero_bindless_descriptor(struct zink_context *ctx, uint32_t handle, bool is_buffer, bool is_image)
1985 {
1986    if (likely(zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor)) {
1987       if (is_buffer) {
1988          VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle];
1989          *bv = VK_NULL_HANDLE;
1990       } else {
1991          VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
1992          memset(ii, 0, sizeof(*ii));
1993       }
1994    } else {
1995       if (is_buffer) {
1996          VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle];
1997          struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
1998          *bv = null_bufferview->buffer_view;
1999       } else {
2000          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
2001          VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
2002          ii->sampler = VK_NULL_HANDLE;
2003          ii->imageView = null_surface->image_view;
2004          ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2005       }
2006    }
2007 }
2008
2009 static void
2010 zink_make_texture_handle_resident(struct pipe_context *pctx, uint64_t handle, bool resident)
2011 {
2012    struct zink_context *ctx = zink_context(pctx);
2013    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2014    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
2015    assert(he);
2016    struct zink_bindless_descriptor *bd = he->data;
2017    struct zink_descriptor_surface *ds = &bd->ds;
2018    struct zink_resource *res = zink_descriptor_surface_resource(ds);
2019    if (is_buffer)
2020       handle -= ZINK_MAX_BINDLESS_HANDLES;
2021    if (resident) {
2022       update_res_bind_count(ctx, res, false, false);
2023       update_res_bind_count(ctx, res, true, false);
2024       res->bindless[0]++;
2025       if (is_buffer) {
2026          if (ds->bufferview->bvci.buffer != res->obj->buffer)
2027             rebind_bindless_bufferview(ctx, res, ds);
2028          VkBufferView *bv = &ctx->di.bindless[0].buffer_infos[handle];
2029          *bv = ds->bufferview->buffer_view;
2030          zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2031          zink_batch_resource_usage_set(&ctx->batch, res, false, true);
2032       } else {
2033          VkDescriptorImageInfo *ii = &ctx->di.bindless[0].img_infos[handle];
2034          ii->sampler = bd->sampler->sampler;
2035          ii->imageView = ds->surface->image_view;
2036          ii->imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
2037          flush_pending_clears(ctx, res);
2038          check_for_layout_update(ctx, res, false);
2039          check_for_layout_update(ctx, res, true);
2040          zink_batch_resource_usage_set(&ctx->batch, res, false, false);
2041          res->obj->unordered_write = false;
2042       }
2043       util_dynarray_append(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2044       uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2045       util_dynarray_append(&ctx->di.bindless[0].updates, uint32_t, h);
2046       res->obj->unordered_read = false;
2047    } else {
2048       zero_bindless_descriptor(ctx, handle, is_buffer, false);
2049       util_dynarray_delete_unordered(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2050       update_res_bind_count(ctx, res, false, true);
2051       update_res_bind_count(ctx, res, true, true);
2052       res->bindless[0]--;
2053       for (unsigned i = 0; i < 2; i++) {
2054          if (!res->image_bind_count[i])
2055             check_for_layout_update(ctx, res, i);
2056       }
2057    }
2058    ctx->di.bindless_dirty[0] = true;
2059 }
2060
2061 static uint64_t
2062 zink_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
2063 {
2064    struct zink_context *ctx = zink_context(pctx);
2065    struct zink_resource *res = zink_resource(view->resource);
2066    struct zink_bindless_descriptor *bd;
2067    if (!zink_resource_object_init_storage(ctx, res)) {
2068       debug_printf("couldn't create storage image!");
2069       return 0;
2070    }
2071    bd = malloc(sizeof(struct zink_bindless_descriptor));
2072    if (!bd)
2073       return 0;
2074    bd->sampler = NULL;
2075
2076    bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
2077    if (res->base.b.target == PIPE_BUFFER)
2078       bd->ds.bufferview = create_image_bufferview(ctx, view);
2079    else
2080       bd->ds.surface = create_image_surface(ctx, view, false);
2081    uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].img_slots);
2082    if (bd->ds.is_buffer)
2083       handle += ZINK_MAX_BINDLESS_HANDLES;
2084    bd->handle = handle;
2085    _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].img_handles, (void*)(uintptr_t)handle, bd);
2086    return handle;
2087 }
2088
2089 static void
2090 zink_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
2091 {
2092    struct zink_context *ctx = zink_context(pctx);
2093    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2094    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2095    assert(he);
2096    struct zink_descriptor_surface *ds = he->data;
2097    _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].img_handles, he);
2098    uint32_t h = handle;
2099    util_dynarray_append(&ctx->batch.state->bindless_releases[1], uint32_t, h);
2100
2101    if (ds->is_buffer) {
2102       zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
2103    } else {
2104       zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
2105    }
2106    free(ds);
2107 }
2108
2109 static void
2110 zink_make_image_handle_resident(struct pipe_context *pctx, uint64_t handle, unsigned paccess, bool resident)
2111 {
2112    struct zink_context *ctx = zink_context(pctx);
2113    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2114    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2115    assert(he);
2116    struct zink_bindless_descriptor *bd = he->data;
2117    struct zink_descriptor_surface *ds = &bd->ds;
2118    bd->access = paccess;
2119    struct zink_resource *res = zink_descriptor_surface_resource(ds);
2120    VkAccessFlags access = 0;
2121    if (paccess & PIPE_IMAGE_ACCESS_WRITE) {
2122       if (resident) {
2123          res->write_bind_count[0]++;
2124          res->write_bind_count[1]++;
2125       } else {
2126          res->write_bind_count[0]--;
2127          res->write_bind_count[1]--;
2128       }
2129       access |= VK_ACCESS_SHADER_WRITE_BIT;
2130    }
2131    if (paccess & PIPE_IMAGE_ACCESS_READ) {
2132       access |= VK_ACCESS_SHADER_READ_BIT;
2133    }
2134    if (is_buffer)
2135       handle -= ZINK_MAX_BINDLESS_HANDLES;
2136    if (resident) {
2137       update_res_bind_count(ctx, res, false, false);
2138       update_res_bind_count(ctx, res, true, false);
2139       res->image_bind_count[0]++;
2140       res->image_bind_count[1]++;
2141       res->bindless[1]++;
2142       if (is_buffer) {
2143          if (ds->bufferview->bvci.buffer != res->obj->buffer)
2144             rebind_bindless_bufferview(ctx, res, ds);
2145          VkBufferView *bv = &ctx->di.bindless[1].buffer_infos[handle];
2146          *bv = ds->bufferview->buffer_view;
2147          zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2148          zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access), true);
2149       } else {
2150          VkDescriptorImageInfo *ii = &ctx->di.bindless[1].img_infos[handle];
2151          ii->sampler = VK_NULL_HANDLE;
2152          ii->imageView = ds->surface->image_view;
2153          ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2154          finalize_image_bind(ctx, res, false);
2155          finalize_image_bind(ctx, res, true);
2156          zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access), false);
2157          res->obj->unordered_write = false;
2158       }
2159       util_dynarray_append(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2160       uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2161       util_dynarray_append(&ctx->di.bindless[1].updates, uint32_t, h);
2162       if (zink_resource_access_is_write(access))
2163          res->obj->unordered_read = res->obj->unordered_write = false;
2164       else
2165          res->obj->unordered_read = false;
2166    } else {
2167       zero_bindless_descriptor(ctx, handle, is_buffer, true);
2168       util_dynarray_delete_unordered(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2169       unbind_shader_image_counts(ctx, res, false, false);
2170       unbind_shader_image_counts(ctx, res, true, false);
2171       res->bindless[1]--;
2172       for (unsigned i = 0; i < 2; i++) {
2173          if (!res->image_bind_count[i])
2174             check_for_layout_update(ctx, res, i);
2175       }
2176    }
2177    ctx->di.bindless_dirty[1] = true;
2178 }
2179
2180 static void
2181 zink_set_stencil_ref(struct pipe_context *pctx,
2182                      const struct pipe_stencil_ref ref)
2183 {
2184    struct zink_context *ctx = zink_context(pctx);
2185    ctx->stencil_ref = ref;
2186    ctx->stencil_ref_changed = true;
2187 }
2188
2189 static void
2190 zink_set_clip_state(struct pipe_context *pctx,
2191                     const struct pipe_clip_state *pcs)
2192 {
2193 }
2194
2195 static void
2196 zink_set_tess_state(struct pipe_context *pctx,
2197                     const float default_outer_level[4],
2198                     const float default_inner_level[2])
2199 {
2200    struct zink_context *ctx = zink_context(pctx);
2201    memcpy(&ctx->default_inner_level, default_inner_level, sizeof(ctx->default_inner_level));
2202    memcpy(&ctx->default_outer_level, default_outer_level, sizeof(ctx->default_outer_level));
2203 }
2204
2205 static void
2206 zink_set_patch_vertices(struct pipe_context *pctx, uint8_t patch_vertices)
2207 {
2208    struct zink_context *ctx = zink_context(pctx);
2209    if (zink_set_tcs_key_patches(ctx, patch_vertices)) {
2210       ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = patch_vertices;
2211       if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints)
2212          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, patch_vertices);
2213       else
2214          ctx->gfx_pipeline_state.dirty = true;
2215    }
2216 }
2217
2218 void
2219 zink_update_fbfetch(struct zink_context *ctx)
2220 {
2221    const bool had_fbfetch = ctx->di.fbfetch.imageLayout == VK_IMAGE_LAYOUT_GENERAL;
2222    if (!ctx->gfx_stages[MESA_SHADER_FRAGMENT] ||
2223        !ctx->gfx_stages[MESA_SHADER_FRAGMENT]->nir->info.fs.uses_fbfetch_output) {
2224       if (!had_fbfetch)
2225          return;
2226       ctx->rp_changed = true;
2227       zink_batch_no_rp(ctx);
2228       ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
2229       ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ?
2230                                   VK_NULL_HANDLE :
2231                                   zink_get_dummy_surface(ctx, 0)->image_view;
2232       zink_context_invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2233       return;
2234    }
2235
2236    bool changed = !had_fbfetch;
2237    if (ctx->fb_state.cbufs[0]) {
2238       VkImageView fbfetch = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2239       if (!fbfetch)
2240          /* swapchain image: retry later */
2241          return;
2242       changed |= fbfetch != ctx->di.fbfetch.imageView;
2243       ctx->di.fbfetch.imageView = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2244
2245       bool fbfetch_ms = ctx->fb_state.cbufs[0]->texture->nr_samples > 1;
2246       if (zink_get_fs_key(ctx)->fbfetch_ms != fbfetch_ms)
2247          zink_set_fs_key(ctx)->fbfetch_ms = fbfetch_ms;
2248    }
2249    ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2250    if (changed) {
2251       zink_context_invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2252       if (!had_fbfetch) {
2253          ctx->rp_changed = true;
2254          zink_batch_no_rp(ctx);
2255       }
2256    }
2257 }
2258
2259 void
2260 zink_update_vk_sample_locations(struct zink_context *ctx)
2261 {
2262    if (ctx->gfx_pipeline_state.sample_locations_enabled && ctx->sample_locations_changed) {
2263       unsigned samples = ctx->gfx_pipeline_state.rast_samples + 1;
2264       unsigned idx = util_logbase2_ceil(MAX2(samples, 1));
2265       VkExtent2D grid_size = zink_screen(ctx->base.screen)->maxSampleLocationGridSize[idx];
2266
2267       for (unsigned pixel = 0; pixel < grid_size.width * grid_size.height; pixel++) {
2268          for (unsigned sample = 0; sample < samples; sample++) {
2269             unsigned pixel_x = pixel % grid_size.width;
2270             unsigned pixel_y = pixel / grid_size.width;
2271             unsigned wi = pixel * samples + sample;
2272             unsigned ri = (pixel_y * grid_size.width + pixel_x % grid_size.width);
2273             ri = ri * samples + sample;
2274             ctx->vk_sample_locations[wi].x = (ctx->sample_locations[ri] & 0xf) / 16.0f;
2275             ctx->vk_sample_locations[wi].y = (16 - (ctx->sample_locations[ri] >> 4)) / 16.0f;
2276          }
2277       }
2278    }
2279 }
2280
2281 static unsigned
2282 find_rp_state(struct zink_context *ctx)
2283 {
2284    bool found = false;
2285    struct set_entry *he = _mesa_set_search_or_add(&ctx->rendering_state_cache, &ctx->gfx_pipeline_state.rendering_info, &found);
2286    struct zink_rendering_info *info;
2287    if (found) {
2288       info = (void*)he->key;
2289       return info->id;
2290    }
2291    info = ralloc(ctx, struct zink_rendering_info);
2292    memcpy(info, &ctx->gfx_pipeline_state.rendering_info, sizeof(VkPipelineRenderingCreateInfo));
2293    info->id = ctx->rendering_state_cache.entries;
2294    he->key = info;
2295    return info->id;
2296 }
2297
2298 static unsigned
2299 begin_rendering(struct zink_context *ctx)
2300 {
2301    unsigned clear_buffers = 0;
2302    ctx->gfx_pipeline_state.render_pass = NULL;
2303    zink_update_vk_sample_locations(ctx);
2304    zink_render_update_swapchain(ctx);
2305    bool has_depth = false;
2306    bool has_stencil = false;
2307    bool changed_layout = false;
2308    bool changed_size = false;
2309    bool zsbuf_used = zink_is_zsbuf_used(ctx);
2310    if (ctx->rp_changed || ctx->rp_layout_changed || ctx->rp_loadop_changed) {
2311       /* init imageviews, base loadOp, formats */
2312       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2313          struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2314
2315          if (!surf || !zink_resource(surf->base.texture)->valid)
2316             ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2317          else
2318             ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2319          if (ctx->tc && zink_screen(ctx->base.screen)->driver_workarounds.track_renderpasses) {
2320             if (ctx->dynamic_fb.tc_info.cbuf_invalidate & BITFIELD_BIT(i))
2321                ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2322             else
2323                ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2324          }
2325          ctx->gfx_pipeline_state.rendering_formats[i] = surf ? surf->info.format[0] : VK_FORMAT_R8G8B8A8_UNORM;
2326          /* use dummy fb size of 1024 if no surf exists */
2327          unsigned width = surf ? surf->base.texture->width0 : 1024;
2328          unsigned height = surf ? surf->base.texture->height0 : 1024;
2329          unsigned prev_width = ctx->dynamic_fb.info.renderArea.extent.width;
2330          unsigned prev_height = ctx->dynamic_fb.info.renderArea.extent.height;
2331          ctx->dynamic_fb.info.renderArea.extent.width = MIN2(ctx->dynamic_fb.info.renderArea.extent.width, width);
2332          ctx->dynamic_fb.info.renderArea.extent.height = MIN2(ctx->dynamic_fb.info.renderArea.extent.height, height);
2333          changed_size |= ctx->dynamic_fb.info.renderArea.extent.width != prev_width;
2334          changed_size |= ctx->dynamic_fb.info.renderArea.extent.height != prev_height;
2335       }
2336
2337       /* unset depth and stencil info: reset below */
2338       VkImageLayout zlayout = ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2339       VkImageLayout slayout = ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2340       ctx->dynamic_fb.info.pDepthAttachment = NULL;
2341       ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED;
2342       ctx->dynamic_fb.info.pStencilAttachment = NULL;
2343       ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = VK_FORMAT_UNDEFINED;
2344
2345       if (ctx->fb_state.zsbuf && zsbuf_used) {
2346          struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2347          has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format));
2348          has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format));
2349
2350          /* depth may or may not be used but init it anyway */
2351          if (zink_resource(surf->base.texture)->valid)
2352             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2353          else
2354             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2355
2356          if (ctx->tc && zink_screen(ctx->base.screen)->driver_workarounds.track_renderpasses) {
2357             if (ctx->dynamic_fb.tc_info.zsbuf_invalidate)
2358                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2359             else
2360                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2361          }
2362
2363          /* stencil may or may not be used but init it anyway */
2364          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp;
2365          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].storeOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp;
2366
2367          if (has_depth) {
2368             ctx->dynamic_fb.info.pDepthAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS];
2369             ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = surf->info.format[0];
2370             /* stencil info only set for clears below */
2371          }
2372          if (has_stencil) {
2373             /* must be stencil-only */
2374             ctx->dynamic_fb.info.pStencilAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1];
2375             ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = surf->info.format[0];
2376          }
2377       } else {
2378          ctx->dynamic_fb.info.pDepthAttachment = NULL;
2379          ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED;
2380       }
2381       if (zlayout != (ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2382          changed_layout = true;
2383       if (slayout != (ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2384          changed_layout = true;
2385
2386       /* similar to begin_render_pass(), but just filling in VkRenderingInfo */
2387       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2388          /* these are no-ops */
2389          if (!ctx->fb_state.cbufs[i] || !zink_fb_clear_enabled(ctx, i))
2390             continue;
2391          /* these need actual clear calls inside the rp */
2392          struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(&ctx->fb_clears[i], 0);
2393          if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i])) {
2394             clear_buffers |= (PIPE_CLEAR_COLOR0 << i);
2395             if (zink_fb_clear_count(&ctx->fb_clears[i]) < 2 ||
2396                 zink_fb_clear_element_needs_explicit(clear))
2397                continue;
2398          }
2399          /* we now know there's one clear that can be done here */
2400          memcpy(&ctx->dynamic_fb.attachments[i].clearValue, &clear->color, sizeof(float) * 4);
2401          ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2402       }
2403       if (ctx->fb_state.zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) {
2404          struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
2405          struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0);
2406          if (!zink_fb_clear_element_needs_explicit(clear)) {
2407             /* base zs clear info */
2408             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.depth = clear->zs.depth;
2409             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.stencil = clear->zs.stencil;
2410             /* always init separate stencil attachment */
2411             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].clearValue.depthStencil.stencil = clear->zs.stencil;
2412             if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_DEPTH))
2413                /* initiate a depth clear */
2414                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2415             if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_STENCIL)) {
2416                /* use a stencil clear, also set stencil attachment */
2417                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2418             }
2419          }
2420          if (zink_fb_clear_needs_explicit(fb_clear)) {
2421             for (int j = !zink_fb_clear_element_needs_explicit(clear);
2422                  (clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL && j < zink_fb_clear_count(fb_clear);
2423                  j++)
2424                clear_buffers |= zink_fb_clear_element(fb_clear, j)->zs.bits;
2425          }
2426       }
2427       if (changed_size || changed_layout)
2428          ctx->rp_changed = true;
2429       ctx->rp_loadop_changed = false;
2430       ctx->rp_layout_changed = false;
2431    }
2432    /* validate zs VUs: attachment must be null or format must be valid */
2433    assert(!ctx->dynamic_fb.info.pDepthAttachment || ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat);
2434    assert(!ctx->dynamic_fb.info.pStencilAttachment || ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat);
2435
2436    if (!ctx->rp_changed && ctx->batch.in_rp)
2437       return 0;
2438    ctx->rp_changed = false;
2439    /* update pipeline info id for compatibility VUs */
2440    unsigned rp_state = find_rp_state(ctx);
2441    bool rp_changed = ctx->gfx_pipeline_state.rp_state != rp_state;
2442    if (!rp_changed && ctx->batch.in_rp)
2443       return 0;
2444    zink_batch_no_rp(ctx);
2445    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2446       struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2447       VkImageView iv = zink_prep_fb_attachment(ctx, surf, i);
2448       if (!iv)
2449          /* dead swapchain */
2450          return 0;
2451       ctx->dynamic_fb.attachments[i].imageView = iv;
2452    }
2453    if (ctx->fb_state.zsbuf && zsbuf_used) {
2454       struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2455       VkImageView iv = zink_prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
2456       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageView = iv;
2457       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout = zink_resource(surf->base.texture)->layout;
2458       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageView = iv;
2459       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout = zink_resource(surf->base.texture)->layout;
2460    }
2461    ctx->gfx_pipeline_state.dirty |= rp_changed;
2462    ctx->gfx_pipeline_state.rp_state = rp_state;
2463
2464    VKCTX(CmdBeginRendering)(ctx->batch.state->cmdbuf, &ctx->dynamic_fb.info);
2465    ctx->batch.in_rp = true;
2466    return clear_buffers;
2467 }
2468
2469 ALWAYS_INLINE static void
2470 update_layered_rendering_state(struct zink_context *ctx)
2471 {
2472    if (!zink_screen(ctx->base.screen)->driver_workarounds.needs_sanitised_layer)
2473       return;
2474    unsigned framebffer_is_layered = zink_framebuffer_get_num_layers(&ctx->fb_state) > 1;
2475    VKCTX(CmdPushConstants)(
2476          ctx->batch.state->cmdbuf,
2477          zink_screen(ctx->base.screen)->gfx_push_constant_layout,
2478          VK_SHADER_STAGE_ALL_GRAPHICS,
2479          offsetof(struct zink_gfx_push_constant, framebuffer_is_layered), sizeof(unsigned),
2480          &framebffer_is_layered);
2481 }
2482
2483 void
2484 zink_batch_rp(struct zink_context *ctx)
2485 {
2486    assert(!(ctx->batch.in_rp && ctx->rp_changed));
2487    if (ctx->batch.in_rp && !ctx->rp_layout_changed)
2488       return;
2489    bool in_rp = ctx->batch.in_rp;
2490    if (!in_rp && ctx->void_clears) {
2491       union pipe_color_union color;
2492       color.f[0] = color.f[1] = color.f[2] = 0;
2493       color.f[3] = 1.0;
2494       ctx->base.clear(&ctx->base, ctx->void_clears, NULL, &color, 0, 0);
2495       ctx->void_clears = 0;
2496    }
2497    update_tc_info(ctx, ctx->rp_tc_info_updated);
2498    ctx->rp_tc_info_updated = false;
2499    unsigned clear_buffers;
2500    /* use renderpass for multisample-to-singlesample or fbfetch:
2501     * - msrtss is TODO
2502     * - dynamic rendering doesn't have input attachments
2503     */
2504    if (!zink_screen(ctx->base.screen)->info.have_KHR_dynamic_rendering || ctx->transient_attachments || ctx->fbfetch_outputs)
2505       clear_buffers = zink_begin_render_pass(ctx);
2506    else
2507       clear_buffers = begin_rendering(ctx);
2508    assert(!ctx->rp_changed);
2509    if (in_rp || !ctx->batch.in_rp)
2510       return; //dead swapchain or continued renderpass
2511    if (ctx->render_condition.query)
2512       zink_start_conditional_render(ctx);
2513    zink_clear_framebuffer(ctx, clear_buffers);
2514 }
2515
2516 void
2517 zink_batch_no_rp(struct zink_context *ctx)
2518 {
2519    if (!ctx->batch.in_rp)
2520       return;
2521    if (ctx->render_condition.query)
2522       zink_stop_conditional_render(ctx);
2523    if (ctx->gfx_pipeline_state.render_pass)
2524       zink_end_render_pass(ctx);
2525    else {
2526       VKCTX(CmdEndRendering)(ctx->batch.state->cmdbuf);
2527       ctx->batch.in_rp = false;
2528    }
2529    assert(!ctx->batch.in_rp);
2530 }
2531
2532 ALWAYS_INLINE static void
2533 update_res_sampler_layouts(struct zink_context *ctx, struct zink_resource *res)
2534 {
2535    unsigned find = res->sampler_bind_count[0];
2536    for (unsigned i = 0; find && i < MESA_SHADER_COMPUTE; i++) {
2537       u_foreach_bit(slot, res->sampler_binds[i]) {
2538          /* only set layout, skip rest of update */
2539          if (ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i][slot] == res)
2540             ctx->di.textures[i][slot].imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
2541          find--;
2542          if (!find) break;
2543       }
2544    }
2545 }
2546
2547 VkImageView
2548 zink_prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i)
2549 {
2550    struct zink_resource *res;
2551    if (!surf || (i < ctx->fb_state.nr_cbufs && zink_use_dummy_attachments(ctx))) {
2552       surf = zink_get_dummy_surface(ctx, util_logbase2_ceil(ctx->fb_state.samples));
2553       res = zink_resource(surf->base.texture);
2554    } else {
2555       res = zink_resource(surf->base.texture);
2556       zink_batch_resource_usage_set(&ctx->batch, res, true, false);
2557    }
2558
2559    VkAccessFlags access;
2560    VkPipelineStageFlags pipeline;
2561    if (zink_is_swapchain(res)) {
2562       if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
2563          return VK_NULL_HANDLE;
2564       zink_surface_swapchain_update(ctx, surf);
2565       if (!i)
2566          zink_update_fbfetch(ctx);
2567    }
2568    VkImageLayout layout;
2569    if (ctx->tc && zink_screen(ctx->base.screen)->driver_workarounds.track_renderpasses && !ctx->blitting) {
2570       assert(threaded_context_get_renderpass_info(ctx->tc, false)->data == ctx->dynamic_fb.tc_info.data);
2571       layout = zink_tc_renderpass_info_parse(ctx, &ctx->dynamic_fb.tc_info, i < ctx->fb_state.nr_cbufs ? i : PIPE_MAX_COLOR_BUFS, &pipeline, &access);
2572       assert(i < ctx->fb_state.nr_cbufs || layout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL || !zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS));
2573       if (i == ctx->fb_state.nr_cbufs && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS))
2574          assert(ctx->dynamic_fb.tc_info.zsbuf_clear || ctx->dynamic_fb.tc_info.zsbuf_clear_partial || ctx->dynamic_fb.tc_info.zsbuf_load);
2575    } else {
2576       if (ctx->gfx_pipeline_state.render_pass) {
2577          layout = zink_render_pass_attachment_get_barrier_info(&ctx->gfx_pipeline_state.render_pass->state.rts[i],
2578                                                                i < ctx->fb_state.nr_cbufs, &pipeline, &access);
2579       } else {
2580          struct zink_rt_attrib rt;
2581          if (i < ctx->fb_state.nr_cbufs)
2582             zink_init_color_attachment(ctx, i, &rt);
2583          else
2584             zink_init_zs_attachment(ctx, &rt);
2585          layout = zink_render_pass_attachment_get_barrier_info(&rt, i < ctx->fb_state.nr_cbufs, &pipeline, &access);
2586       }
2587    }
2588    if (!zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_layout &&
2589        layout == VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT)
2590       layout = VK_IMAGE_LAYOUT_GENERAL;
2591    zink_screen(ctx->base.screen)->image_barrier(ctx, res, layout, access, pipeline);
2592    res->obj->unordered_read = res->obj->unordered_write = false;
2593    if (i == ctx->fb_state.nr_cbufs && res->sampler_bind_count[0])
2594       update_res_sampler_layouts(ctx, res);
2595    return surf->image_view;
2596 }
2597
2598 static uint32_t
2599 hash_rendering_state(const void *key)
2600 {
2601    const VkPipelineRenderingCreateInfo *info = key;
2602    uint32_t hash = 0;
2603    /*
2604     uint32_t           colorAttachmentCount;
2605     const VkFormat*    pColorAttachmentFormats;
2606     VkFormat           depthAttachmentFormat;
2607     VkFormat           stencilAttachmentFormat;
2608     * this data is not optimally arranged, so it must be manually hashed
2609     */
2610    hash = XXH32(&info->colorAttachmentCount, sizeof(uint32_t), hash);
2611    hash = XXH32(&info->depthAttachmentFormat, sizeof(uint32_t), hash);
2612    hash = XXH32(&info->stencilAttachmentFormat, sizeof(VkFormat), hash);
2613    return XXH32(info->pColorAttachmentFormats, sizeof(VkFormat) * info->colorAttachmentCount, hash);
2614 }
2615
2616 static bool
2617 equals_rendering_state(const void *a, const void *b)
2618 {
2619    const VkPipelineRenderingCreateInfo *ai = a;
2620    const VkPipelineRenderingCreateInfo *bi = b;
2621    return ai->colorAttachmentCount == bi->colorAttachmentCount &&
2622           ai->depthAttachmentFormat == bi->depthAttachmentFormat &&
2623           ai->stencilAttachmentFormat == bi->stencilAttachmentFormat &&
2624           !memcmp(ai->pColorAttachmentFormats, bi->pColorAttachmentFormats, sizeof(VkFormat) * ai->colorAttachmentCount);
2625 }
2626
2627 static uint32_t
2628 hash_framebuffer_imageless(const void *key)
2629 {
2630    struct zink_framebuffer_state* s = (struct zink_framebuffer_state*)key;
2631    return _mesa_hash_data(key, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments);
2632 }
2633
2634 static bool
2635 equals_framebuffer_imageless(const void *a, const void *b)
2636 {
2637    struct zink_framebuffer_state *s = (struct zink_framebuffer_state*)a;
2638    return memcmp(a, b, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments) == 0;
2639 }
2640
2641 void
2642 zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc)
2643 {
2644    struct zink_screen *screen = zink_screen(ctx->base.screen);
2645    unsigned idx = util_logbase2_ceil(MAX2(ctx->gfx_pipeline_state.rast_samples + 1, 1));
2646    loc->sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT;
2647    loc->pNext = NULL;
2648    loc->sampleLocationsPerPixel = 1 << idx;
2649    loc->sampleLocationsCount = ctx->gfx_pipeline_state.rast_samples + 1;
2650    loc->sampleLocationGridSize = screen->maxSampleLocationGridSize[idx];
2651    loc->pSampleLocations = ctx->vk_sample_locations;
2652 }
2653
2654 static void
2655 zink_evaluate_depth_buffer(struct pipe_context *pctx)
2656 {
2657    struct zink_context *ctx = zink_context(pctx);
2658
2659    if (!ctx->fb_state.zsbuf)
2660       return;
2661
2662    struct zink_resource *res = zink_resource(ctx->fb_state.zsbuf->texture);
2663    res->obj->needs_zs_evaluate = true;
2664    zink_init_vk_sample_locations(ctx, &res->obj->zs_evaluate);
2665    zink_batch_no_rp(ctx);
2666 }
2667
2668 static void
2669 sync_flush(struct zink_context *ctx, struct zink_batch_state *bs)
2670 {
2671    if (zink_screen(ctx->base.screen)->threaded)
2672       util_queue_fence_wait(&bs->flush_completed);
2673 }
2674
2675 static inline VkAccessFlags
2676 get_access_flags_for_binding(struct zink_context *ctx, enum zink_descriptor_type type, gl_shader_stage stage, unsigned idx)
2677 {
2678    VkAccessFlags flags = 0;
2679    switch (type) {
2680    case ZINK_DESCRIPTOR_TYPE_UBO:
2681       return VK_ACCESS_UNIFORM_READ_BIT;
2682    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
2683       return VK_ACCESS_SHADER_READ_BIT;
2684    case ZINK_DESCRIPTOR_TYPE_SSBO: {
2685       flags = VK_ACCESS_SHADER_READ_BIT;
2686       if (ctx->writable_ssbos[stage] & (1 << idx))
2687          flags |= VK_ACCESS_SHADER_WRITE_BIT;
2688       return flags;
2689    }
2690    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
2691       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
2692       if (image_view->base.access & PIPE_IMAGE_ACCESS_READ)
2693          flags |= VK_ACCESS_SHADER_READ_BIT;
2694       if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE)
2695          flags |= VK_ACCESS_SHADER_WRITE_BIT;
2696       return flags;
2697    }
2698    default:
2699       break;
2700    }
2701    unreachable("ACK");
2702    return 0;
2703 }
2704
2705 static void
2706 update_resource_refs_for_stage(struct zink_context *ctx, gl_shader_stage stage)
2707 {
2708    struct zink_batch *batch = &ctx->batch;
2709    unsigned max_slot[] = {
2710       [ZINK_DESCRIPTOR_TYPE_UBO] = ctx->di.num_ubos[stage],
2711       [ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = ctx->di.num_samplers[stage],
2712       [ZINK_DESCRIPTOR_TYPE_SSBO] = ctx->di.num_ssbos[stage],
2713       [ZINK_DESCRIPTOR_TYPE_IMAGE] = ctx->di.num_images[stage]
2714    };
2715    for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) {
2716       for (unsigned j = 0; j < max_slot[i]; j++) {
2717          if (ctx->di.descriptor_res[i][stage][j]) {
2718             struct zink_resource *res = ctx->di.descriptor_res[i][stage][j];
2719             if (!res)
2720                continue;
2721             bool is_buffer = res->obj->is_buffer;
2722             bool is_write = zink_resource_access_is_write(get_access_flags_for_binding(ctx, i, stage, j));
2723             zink_batch_resource_usage_set(batch, res, is_write, is_buffer);
2724             if (is_write || !res->obj->is_buffer)
2725                res->obj->unordered_read = res->obj->unordered_write = false;
2726             else
2727                res->obj->unordered_read = false;
2728          }
2729       }
2730    }
2731 }
2732
2733 void
2734 zink_update_descriptor_refs(struct zink_context *ctx, bool compute)
2735 {
2736    struct zink_batch *batch = &ctx->batch;
2737    if (compute) {
2738       update_resource_refs_for_stage(ctx, MESA_SHADER_COMPUTE);
2739       if (ctx->curr_compute)
2740          zink_batch_reference_program(batch, &ctx->curr_compute->base);
2741    } else {
2742       for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++)
2743          update_resource_refs_for_stage(ctx, i);
2744       unsigned vertex_buffers_enabled_mask = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
2745       unsigned last_vbo = util_last_bit(vertex_buffers_enabled_mask);
2746       for (unsigned i = 0; i < last_vbo + 1; i++) {
2747          struct zink_resource *res = zink_resource(ctx->vertex_buffers[i].buffer.resource);
2748          if (res) {
2749             zink_batch_resource_usage_set(batch, res, false, true);
2750             res->obj->unordered_read = false;
2751          }
2752       }
2753       if (ctx->curr_program)
2754          zink_batch_reference_program(batch, &ctx->curr_program->base);
2755    }
2756    if (ctx->di.bindless_refs_dirty) {
2757       ctx->di.bindless_refs_dirty = false;
2758       for (unsigned i = 0; i < 2; i++) {
2759          util_dynarray_foreach(&ctx->di.bindless[i].resident, struct zink_bindless_descriptor*, bd) {
2760             struct zink_resource *res = zink_descriptor_surface_resource(&(*bd)->ds);
2761             zink_batch_resource_usage_set(&ctx->batch, res, (*bd)->access & PIPE_IMAGE_ACCESS_WRITE, res->obj->is_buffer);
2762             if ((*bd)->access & PIPE_IMAGE_ACCESS_WRITE || !res->obj->is_buffer)
2763                res->obj->unordered_read = res->obj->unordered_write = false;
2764             else
2765                res->obj->unordered_read = false;
2766          }
2767       }
2768    }
2769 }
2770
2771 static void
2772 reapply_color_write(struct zink_context *ctx)
2773 {
2774    struct zink_screen *screen = zink_screen(ctx->base.screen);
2775    if (screen->info.have_EXT_color_write_enable) {
2776       const VkBool32 enables[PIPE_MAX_COLOR_BUFS] = {1, 1, 1, 1, 1, 1, 1, 1};
2777       const VkBool32 disables[PIPE_MAX_COLOR_BUFS] = {0};
2778       const unsigned max_att = MIN2(PIPE_MAX_COLOR_BUFS, screen->info.props.limits.maxColorAttachments);
2779       VKCTX(CmdSetColorWriteEnableEXT)(ctx->batch.state->cmdbuf, max_att, ctx->disable_color_writes ? disables : enables);
2780    }
2781    if (screen->info.have_EXT_extended_dynamic_state && ctx->dsa_state)
2782       VKCTX(CmdSetDepthWriteEnableEXT)(ctx->batch.state->cmdbuf, ctx->disable_color_writes ? VK_FALSE : ctx->dsa_state->hw_state.depth_write);
2783 }
2784
2785 static void
2786 stall(struct zink_context *ctx)
2787 {
2788    struct zink_screen *screen = zink_screen(ctx->base.screen);
2789    sync_flush(ctx, zink_batch_state(ctx->last_fence));
2790    zink_screen_timeline_wait(screen, ctx->last_fence->batch_id, PIPE_TIMEOUT_INFINITE);
2791    zink_batch_reset_all(ctx);
2792 }
2793
2794 static void
2795 flush_batch(struct zink_context *ctx, bool sync)
2796 {
2797    struct zink_batch *batch = &ctx->batch;
2798    if (ctx->clears_enabled)
2799       /* start rp to do all the clears */
2800       zink_batch_rp(ctx);
2801    bool conditional_render_active = ctx->render_condition.active;
2802    zink_stop_conditional_render(ctx);
2803    zink_batch_no_rp(ctx);
2804    zink_end_batch(ctx, batch);
2805    ctx->deferred_fence = NULL;
2806
2807    if (sync)
2808       sync_flush(ctx, ctx->batch.state);
2809
2810    if (ctx->batch.state->is_device_lost) {
2811       check_device_lost(ctx);
2812    } else {
2813       zink_start_batch(ctx, batch);
2814       if (zink_screen(ctx->base.screen)->info.have_EXT_transform_feedback && ctx->num_so_targets)
2815          ctx->dirty_so_targets = true;
2816       ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
2817       zink_select_draw_vbo(ctx);
2818       zink_select_launch_grid(ctx);
2819
2820       if (ctx->oom_stall)
2821          stall(ctx);
2822       ctx->oom_flush = false;
2823       ctx->oom_stall = false;
2824       ctx->dd.bindless_bound = false;
2825       ctx->di.bindless_refs_dirty = true;
2826       ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
2827       if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints)
2828          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch);
2829       if (conditional_render_active)
2830          zink_start_conditional_render(ctx);
2831       reapply_color_write(ctx);
2832       update_layered_rendering_state(ctx);
2833    }
2834 }
2835
2836 void
2837 zink_flush_queue(struct zink_context *ctx)
2838 {
2839    flush_batch(ctx, true);
2840 }
2841
2842 static bool
2843 rebind_fb_surface(struct zink_context *ctx, struct pipe_surface **surf, struct zink_resource *match_res)
2844 {
2845    if (!*surf)
2846       return false;
2847    struct zink_resource *surf_res = zink_resource((*surf)->texture);
2848    if ((match_res == surf_res) || surf_res->obj != zink_csurface(*surf)->obj)
2849       return zink_rebind_ctx_surface(ctx, surf);
2850    return false;
2851 }
2852
2853 static bool
2854 rebind_fb_state(struct zink_context *ctx, struct zink_resource *match_res, bool from_set_fb)
2855 {
2856    bool rebind = false;
2857    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++)
2858       rebind |= rebind_fb_surface(ctx, &ctx->fb_state.cbufs[i], match_res);
2859    rebind |= rebind_fb_surface(ctx, &ctx->fb_state.zsbuf, match_res);
2860    return rebind;
2861 }
2862
2863 static void
2864 unbind_fb_surface(struct zink_context *ctx, struct pipe_surface *surf, unsigned idx, bool changed)
2865 {
2866    ctx->dynamic_fb.attachments[idx].imageView = VK_NULL_HANDLE;
2867    if (!surf)
2868       return;
2869    struct zink_resource *res = zink_resource(surf->texture);
2870    if (changed) {
2871       ctx->rp_changed = true;
2872    }
2873    res->fb_bind_count--;
2874    unsigned feedback_loops = ctx->feedback_loops;
2875    if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
2876       ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
2877       ctx->rp_layout_changed = true;
2878    }
2879    ctx->feedback_loops &= ~BITFIELD_BIT(idx);
2880    if (feedback_loops != ctx->feedback_loops) {
2881       if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
2882          if (ctx->gfx_pipeline_state.feedback_loop_zs)
2883             ctx->gfx_pipeline_state.dirty = true;
2884          ctx->gfx_pipeline_state.feedback_loop_zs = false;
2885       } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
2886          if (ctx->gfx_pipeline_state.feedback_loop)
2887             ctx->gfx_pipeline_state.dirty = true;
2888          ctx->gfx_pipeline_state.feedback_loop = false;
2889       }
2890    }
2891    res->fb_binds &= ~BITFIELD_BIT(idx);
2892    if (!res->fb_bind_count) {
2893       check_resource_for_batch_ref(ctx, res);
2894       if (res->sampler_bind_count[0]) {
2895          update_res_sampler_layouts(ctx, res);
2896          _mesa_set_add(ctx->need_barriers[0], res);
2897       }
2898    }
2899 }
2900
2901 void
2902 zink_set_color_write_enables(struct zink_context *ctx)
2903 {
2904    bool disable_color_writes = ctx->rast_state && ctx->rast_state->base.rasterizer_discard && ctx->primitives_generated_active;
2905    if (ctx->disable_color_writes == disable_color_writes)
2906       return;
2907    /* flush all pending clears: these have already occurred */
2908    if (disable_color_writes && ctx->clears_enabled)
2909       zink_batch_rp(ctx);
2910    ctx->disable_color_writes = disable_color_writes;
2911    if (!zink_screen(ctx->base.screen)->info.have_EXT_color_write_enable) {
2912       /* use dummy color buffers instead of the more sane option */
2913       zink_batch_no_rp(ctx);
2914       ctx->rp_changed = true;
2915       zink_update_framebuffer_state(ctx);
2916    } else {
2917       reapply_color_write(ctx);
2918    }
2919 }
2920
2921 static void
2922 zink_set_framebuffer_state(struct pipe_context *pctx,
2923                            const struct pipe_framebuffer_state *state)
2924 {
2925    struct zink_context *ctx = zink_context(pctx);
2926    struct zink_screen *screen = zink_screen(pctx->screen);
2927    unsigned samples = state->nr_cbufs || state->zsbuf ? 0 : state->samples;
2928    unsigned w = ctx->fb_state.width;
2929    unsigned h = ctx->fb_state.height;
2930    unsigned layers = MAX2(zink_framebuffer_get_num_layers(state), 1);
2931
2932    bool flush_clears = ctx->clears_enabled &&
2933                        (ctx->dynamic_fb.info.layerCount != layers ||
2934                         state->width != w || state->height != h);
2935    if (ctx->clears_enabled && !flush_clears) {
2936       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2937          if (i >= state->nr_cbufs || !ctx->fb_state.cbufs[i] || !state->cbufs[i])
2938             flush_clears |= zink_fb_clear_enabled(ctx, i);
2939          else if (zink_fb_clear_enabled(ctx, i) && ctx->fb_state.cbufs[i] != state->cbufs[i]) {
2940             struct zink_surface *a = zink_csurface(ctx->fb_state.cbufs[i]);
2941             struct zink_surface *b = zink_csurface(state->cbufs[i]);
2942             if (a == b)
2943                continue;
2944             if (memcmp(&a->base.u.tex, &b->base.u.tex, sizeof(b->base.u.tex)) ||
2945                 a->base.texture != b->base.texture)
2946                flush_clears = true;
2947             else if (a->base.format != b->base.format)
2948                zink_fb_clear_rewrite(ctx, i, a->base.format, b->base.format);
2949          }
2950       }
2951    }
2952    if (ctx->fb_state.zsbuf != state->zsbuf)
2953       flush_clears |= zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS);
2954    if (flush_clears)
2955       zink_batch_rp(ctx);
2956    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2957       struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
2958       if (i < state->nr_cbufs)
2959          ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->cbufs[i]);
2960       unbind_fb_surface(ctx, psurf, i, i >= state->nr_cbufs || psurf != state->cbufs[i]);
2961       if (psurf && ctx->needs_present == zink_resource(psurf->texture))
2962          ctx->needs_present = NULL;
2963    }
2964    if (ctx->fb_state.zsbuf) {
2965       struct pipe_surface *psurf = ctx->fb_state.zsbuf;
2966       struct zink_resource *res = zink_resource(psurf->texture);
2967       bool changed = psurf != state->zsbuf;
2968       unbind_fb_surface(ctx, psurf, PIPE_MAX_COLOR_BUFS, changed);
2969       if (!changed)
2970          ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->zsbuf);
2971       if (changed && unlikely(res->obj->needs_zs_evaluate))
2972          /* have to flush zs eval while the sample location data still exists,
2973           * so just throw some random barrier */
2974          zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
2975                                      VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
2976    }
2977    /* renderpass changes if the number or types of attachments change */
2978    ctx->rp_changed |= ctx->fb_state.nr_cbufs != state->nr_cbufs;
2979    ctx->rp_changed |= !!ctx->fb_state.zsbuf != !!state->zsbuf;
2980    if (ctx->fb_state.nr_cbufs != state->nr_cbufs)
2981       ctx->blend_state_changed |= screen->have_full_ds3;
2982
2983    util_copy_framebuffer_state(&ctx->fb_state, state);
2984    zink_update_fbfetch(ctx);
2985    ctx->transient_attachments = 0;
2986    ctx->fb_layer_mismatch = 0;
2987
2988    ctx->dynamic_fb.info.renderArea.offset.x = 0;
2989    ctx->dynamic_fb.info.renderArea.offset.y = 0;
2990    ctx->dynamic_fb.info.renderArea.extent.width = state->width;
2991    ctx->dynamic_fb.info.renderArea.extent.height = state->height;
2992    ctx->dynamic_fb.info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
2993    ctx->rp_changed |= ctx->dynamic_fb.info.layerCount != layers;
2994    ctx->dynamic_fb.info.layerCount = layers;
2995    ctx->gfx_pipeline_state.rendering_info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
2996
2997    ctx->void_clears = 0;
2998    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2999       struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
3000       if (psurf) {
3001          struct zink_surface *transient = zink_transient_surface(psurf);
3002          if (transient)
3003             ctx->transient_attachments |= BITFIELD_BIT(i);
3004          if (!samples)
3005             samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, 1);
3006          struct zink_resource *res = zink_resource(psurf->texture);
3007          if (zink_csurface(psurf)->info.layerCount > layers)
3008             ctx->fb_layer_mismatch |= BITFIELD_BIT(i);
3009          if (res->modifiers) {
3010             assert(!ctx->needs_present || ctx->needs_present == res);
3011             ctx->needs_present = res;
3012          }
3013          if (res->obj->dt) {
3014             /* #6274 */
3015             if (!zink_screen(ctx->base.screen)->info.have_KHR_swapchain_mutable_format &&
3016                 psurf->format != res->base.b.format) {
3017                static bool warned = false;
3018                if (!warned) {
3019                   mesa_loge("zink: SRGB framebuffer unsupported without KHR_swapchain_mutable_format");
3020                   warned = true;
3021                }
3022             }
3023          }
3024          res->fb_bind_count++;
3025          res->fb_binds |= BITFIELD_BIT(i);
3026          if (util_format_has_alpha1(psurf->format)) {
3027             if (!res->valid && !zink_fb_clear_full_exists(ctx, i))
3028                ctx->void_clears |= (PIPE_CLEAR_COLOR0 << i);
3029          }
3030       }
3031    }
3032    unsigned depth_bias_scale_factor = ctx->depth_bias_scale_factor;
3033    if (ctx->fb_state.zsbuf) {
3034       struct pipe_surface *psurf = ctx->fb_state.zsbuf;
3035       struct zink_surface *transient = zink_transient_surface(psurf);
3036       if (transient)
3037          ctx->transient_attachments |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3038       if (!samples)
3039          samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, 1);
3040       if (zink_csurface(psurf)->info.layerCount > layers)
3041          ctx->fb_layer_mismatch |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3042       zink_resource(psurf->texture)->fb_bind_count++;
3043       zink_resource(psurf->texture)->fb_binds |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3044       switch (psurf->format) {
3045       case PIPE_FORMAT_Z16_UNORM:
3046       case PIPE_FORMAT_Z16_UNORM_S8_UINT:
3047          ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z16_unscaled_bias;
3048          break;
3049       case PIPE_FORMAT_Z24X8_UNORM:
3050       case PIPE_FORMAT_Z24_UNORM_S8_UINT:
3051       case PIPE_FORMAT_X24S8_UINT:
3052       case PIPE_FORMAT_X8Z24_UNORM:
3053          ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z24_unscaled_bias;
3054          break;
3055       case PIPE_FORMAT_Z32_FLOAT:
3056       case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
3057       case PIPE_FORMAT_Z32_UNORM:
3058          ctx->depth_bias_scale_factor = 1<<23;
3059          break;
3060       default:
3061          ctx->depth_bias_scale_factor = 0;
3062       }
3063    } else {
3064       ctx->depth_bias_scale_factor = 0;
3065    }
3066    if (depth_bias_scale_factor != ctx->depth_bias_scale_factor &&
3067        ctx->rast_state && ctx->rast_state->base.offset_units_unscaled)
3068       ctx->rast_state_changed = true;
3069    rebind_fb_state(ctx, NULL, true);
3070    ctx->fb_state.samples = MAX2(samples, 1);
3071    zink_update_framebuffer_state(ctx);
3072    if (ctx->fb_state.width != w || ctx->fb_state.height != h)
3073       ctx->scissor_changed = true;
3074
3075    uint8_t rast_samples = ctx->fb_state.samples - 1;
3076    if (rast_samples != ctx->gfx_pipeline_state.rast_samples)
3077       zink_update_fs_key_samples(ctx);
3078    if (ctx->gfx_pipeline_state.rast_samples != rast_samples) {
3079       ctx->sample_locations_changed |= ctx->gfx_pipeline_state.sample_locations_enabled;
3080       if (screen->have_full_ds3)
3081          ctx->sample_mask_changed = true;
3082       else
3083          ctx->gfx_pipeline_state.dirty = true;
3084    }
3085    ctx->gfx_pipeline_state.rast_samples = rast_samples;
3086
3087    /* need to ensure we start a new rp on next draw */
3088    zink_batch_no_rp(ctx);
3089    /* this is an ideal time to oom flush since it won't split a renderpass */
3090    if (ctx->oom_flush)
3091       flush_batch(ctx, false);
3092    else
3093       update_layered_rendering_state(ctx);
3094
3095    ctx->rp_tc_info_updated = !ctx->blitting;
3096 }
3097
3098 static void
3099 zink_set_blend_color(struct pipe_context *pctx,
3100                      const struct pipe_blend_color *color)
3101 {
3102    struct zink_context *ctx = zink_context(pctx);
3103    memcpy(ctx->blend_constants, color->color, sizeof(float) * 4);
3104 }
3105
3106 static void
3107 zink_set_sample_mask(struct pipe_context *pctx, unsigned sample_mask)
3108 {
3109    struct zink_context *ctx = zink_context(pctx);
3110    ctx->gfx_pipeline_state.sample_mask = sample_mask;
3111    if (zink_screen(pctx->screen)->have_full_ds3)
3112       ctx->sample_mask_changed = true;
3113    else
3114       ctx->gfx_pipeline_state.dirty = true;
3115 }
3116
3117 static void
3118 zink_set_min_samples(struct pipe_context *pctx, unsigned min_samples)
3119 {
3120    struct zink_context *ctx = zink_context(pctx);
3121    ctx->gfx_pipeline_state.min_samples = min_samples - 1;
3122    ctx->gfx_pipeline_state.dirty = true;
3123 }
3124
3125 static void
3126 zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t *locations)
3127 {
3128    struct zink_context *ctx = zink_context(pctx);
3129
3130    ctx->gfx_pipeline_state.sample_locations_enabled = size && locations;
3131    ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
3132    if (size > sizeof(ctx->sample_locations))
3133       size = sizeof(ctx->sample_locations);
3134
3135    if (locations)
3136       memcpy(ctx->sample_locations, locations, size);
3137 }
3138
3139 static VkAccessFlags
3140 access_src_flags(VkImageLayout layout)
3141 {
3142    switch (layout) {
3143    case VK_IMAGE_LAYOUT_UNDEFINED:
3144       return 0;
3145
3146    case VK_IMAGE_LAYOUT_GENERAL:
3147       return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
3148
3149    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
3150    case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
3151       return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
3152    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
3153       return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
3154
3155    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
3156    case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
3157       return VK_ACCESS_SHADER_READ_BIT;
3158
3159    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
3160       return VK_ACCESS_TRANSFER_READ_BIT;
3161
3162    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
3163       return VK_ACCESS_TRANSFER_WRITE_BIT;
3164
3165    case VK_IMAGE_LAYOUT_PREINITIALIZED:
3166       return VK_ACCESS_HOST_WRITE_BIT;
3167
3168    case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
3169       return 0;
3170
3171    default:
3172       unreachable("unexpected layout");
3173    }
3174 }
3175
3176 static VkAccessFlags
3177 access_dst_flags(VkImageLayout layout)
3178 {
3179    switch (layout) {
3180    case VK_IMAGE_LAYOUT_UNDEFINED:
3181       return 0;
3182
3183    case VK_IMAGE_LAYOUT_GENERAL:
3184       return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
3185
3186    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
3187    case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
3188       return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3189    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
3190       return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
3191
3192    case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
3193       return VK_ACCESS_SHADER_READ_BIT;
3194
3195    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
3196       return VK_ACCESS_TRANSFER_READ_BIT;
3197
3198    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
3199       return VK_ACCESS_SHADER_READ_BIT;
3200
3201    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
3202       return VK_ACCESS_TRANSFER_WRITE_BIT;
3203
3204    case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
3205       return 0;
3206
3207    default:
3208       unreachable("unexpected layout");
3209    }
3210 }
3211
3212 static VkPipelineStageFlags
3213 pipeline_dst_stage(VkImageLayout layout)
3214 {
3215    switch (layout) {
3216    case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
3217       return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
3218    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
3219       return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
3220
3221    case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
3222       return VK_PIPELINE_STAGE_TRANSFER_BIT;
3223    case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
3224       return VK_PIPELINE_STAGE_TRANSFER_BIT;
3225
3226    case VK_IMAGE_LAYOUT_GENERAL:
3227       return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
3228
3229    case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
3230    case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
3231       return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
3232
3233    default:
3234       return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
3235    }
3236 }
3237
3238 #define ALL_READ_ACCESS_FLAGS \
3239     (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \
3240     VK_ACCESS_INDEX_READ_BIT | \
3241     VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \
3242     VK_ACCESS_UNIFORM_READ_BIT | \
3243     VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \
3244     VK_ACCESS_SHADER_READ_BIT | \
3245     VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \
3246     VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \
3247     VK_ACCESS_TRANSFER_READ_BIT |\
3248     VK_ACCESS_HOST_READ_BIT |\
3249     VK_ACCESS_MEMORY_READ_BIT |\
3250     VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\
3251     VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\
3252     VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\
3253     VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
3254     VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\
3255     VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\
3256     VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\
3257     VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
3258     VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR)
3259
3260
3261 bool
3262 zink_resource_access_is_write(VkAccessFlags flags)
3263 {
3264    return (flags & ALL_READ_ACCESS_FLAGS) != flags;
3265 }
3266
3267 bool
3268 zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3269 {
3270    if (!pipeline)
3271       pipeline = pipeline_dst_stage(new_layout);
3272    if (!flags)
3273       flags = access_dst_flags(new_layout);
3274    return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline ||
3275           (res->obj->access & flags) != flags ||
3276           zink_resource_access_is_write(res->obj->access) ||
3277           zink_resource_access_is_write(flags);
3278 }
3279
3280 bool
3281 zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3282 {
3283    if (!pipeline)
3284       pipeline = pipeline_dst_stage(new_layout);
3285    if (!flags)
3286       flags = access_dst_flags(new_layout);
3287
3288    VkImageSubresourceRange isr = {
3289       res->aspect,
3290       0, VK_REMAINING_MIP_LEVELS,
3291       0, VK_REMAINING_ARRAY_LAYERS
3292    };
3293    *imb = (VkImageMemoryBarrier){
3294       VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
3295       NULL,
3296       res->obj->access ? res->obj->access : access_src_flags(res->layout),
3297       flags,
3298       res->layout,
3299       new_layout,
3300       VK_QUEUE_FAMILY_IGNORED,
3301       VK_QUEUE_FAMILY_IGNORED,
3302       res->obj->image,
3303       isr
3304    };
3305    return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline);
3306 }
3307
3308 static bool
3309 zink_resource_image_barrier2_init(VkImageMemoryBarrier2 *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3310 {
3311    if (!pipeline)
3312       pipeline = pipeline_dst_stage(new_layout);
3313    if (!flags)
3314       flags = access_dst_flags(new_layout);
3315
3316    VkImageSubresourceRange isr = {
3317       res->aspect,
3318       0, VK_REMAINING_MIP_LEVELS,
3319       0, VK_REMAINING_ARRAY_LAYERS
3320    };
3321    *imb = (VkImageMemoryBarrier2){
3322       VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
3323       NULL,
3324       res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
3325       res->obj->access ? res->obj->access : access_src_flags(res->layout),
3326       pipeline,
3327       flags,
3328       res->layout,
3329       new_layout,
3330       VK_QUEUE_FAMILY_IGNORED,
3331       VK_QUEUE_FAMILY_IGNORED,
3332       res->obj->image,
3333       isr
3334    };
3335    return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline);
3336 }
3337
3338 static inline bool
3339 is_shader_pipline_stage(VkPipelineStageFlags pipeline)
3340 {
3341    return pipeline & GFX_SHADER_BITS;
3342 }
3343
3344 static void
3345 resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline)
3346 {
3347    assert(res->obj->is_buffer);
3348    if (res->bind_count[0] - res->so_bind_count > 0) {
3349       if ((res->obj->is_buffer && res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) ||
3350           ((!res->obj->is_buffer || util_bitcount(res->vbo_bind_mask) != res->bind_count[0]) && !is_shader_pipline_stage(pipeline)))
3351          /* gfx rebind */
3352          _mesa_set_add(ctx->need_barriers[0], res);
3353    }
3354    if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT))
3355       /* compute rebind */
3356       _mesa_set_add(ctx->need_barriers[1], res);
3357 }
3358
3359 static inline bool
3360 unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write)
3361 {
3362    /* if all usage is unordered, keep unordered */
3363    if (res->obj->unordered_read && res->obj->unordered_write)
3364       return true;
3365    /* if testing write access but have any ordered read access, cannot promote */
3366    if (is_write && zink_batch_usage_matches(res->obj->bo->reads, ctx->batch.state) && !res->obj->unordered_read)
3367       return false;
3368    /* if write access is unordered or nonexistent, always promote */
3369    return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes, ctx->batch.state);
3370 }
3371
3372 VkCommandBuffer
3373 zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst)
3374 {
3375    bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0;
3376    if (src)
3377       unordered_exec &= unordered_res_exec(ctx, src, false);
3378    if (dst)
3379       unordered_exec &= unordered_res_exec(ctx, dst, true);
3380    if (src)
3381       src->obj->unordered_read = unordered_exec;
3382    if (dst)
3383       dst->obj->unordered_write = unordered_exec;
3384    if (unordered_exec) {
3385       ctx->batch.state->has_barriers = true;
3386       return ctx->batch.state->barrier_cmdbuf;
3387    }
3388    zink_batch_no_rp(ctx);
3389    return ctx->batch.state->cmdbuf;
3390 }
3391
3392 static void
3393 resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline)
3394 {
3395    assert(!res->obj->is_buffer);
3396
3397    bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
3398    /* if this is a non-shader barrier and there are binds, always queue a shader barrier */
3399    bool is_shader = is_shader_pipline_stage(pipeline);
3400    if ((is_shader || !res->bind_count[is_compute]) &&
3401        /* if no layout change is needed between gfx and compute, do nothing */
3402        !res->bind_count[!is_compute] && (!is_compute || !res->fb_bind_count))
3403       return;
3404
3405    if (res->bind_count[!is_compute] && is_shader) {
3406       /* if the layout is the same between gfx and compute, do nothing */
3407       if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute))
3408          return;
3409    }
3410    /* queue a layout change if a layout change will be needed */
3411    if (res->bind_count[!is_compute])
3412       _mesa_set_add(ctx->need_barriers[!is_compute], res);
3413    /* also queue a layout change if this is a non-shader layout */
3414    if (res->bind_count[is_compute] && !is_shader)
3415       _mesa_set_add(ctx->need_barriers[is_compute], res);
3416 }
3417
3418 void
3419 zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
3420                       VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3421 {
3422    VkImageMemoryBarrier imb;
3423    if (!pipeline)
3424       pipeline = pipeline_dst_stage(new_layout);
3425
3426    if (!zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline))
3427       return;
3428    /* only barrier if we're changing layout or doing something besides read -> read */
3429    bool is_write = zink_resource_access_is_write(imb.dstAccessMask);
3430    VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
3431    assert(new_layout);
3432    if (!res->obj->access_stage)
3433       imb.srcAccessMask = 0;
3434    if (res->obj->needs_zs_evaluate)
3435       imb.pNext = &res->obj->zs_evaluate;
3436    res->obj->needs_zs_evaluate = false;
3437    if (res->dmabuf_acquire) {
3438       imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
3439       imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
3440       res->dmabuf_acquire = false;
3441    }
3442    VKCTX(CmdPipelineBarrier)(
3443       cmdbuf,
3444       res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
3445       pipeline,
3446       0,
3447       0, NULL,
3448       0, NULL,
3449       1, &imb
3450    );
3451
3452    resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
3453
3454    res->obj->access = imb.dstAccessMask;
3455    res->obj->access_stage = pipeline;
3456    res->layout = new_layout;
3457 }
3458
3459 void
3460 zink_resource_image_barrier2(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3461 {
3462    VkImageMemoryBarrier2 imb;
3463    if (!pipeline)
3464       pipeline = pipeline_dst_stage(new_layout);
3465
3466    if (!zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline))
3467       return;
3468    /* only barrier if we're changing layout or doing something besides read -> read */
3469    bool is_write = zink_resource_access_is_write(imb.dstAccessMask);
3470    VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
3471    assert(new_layout);
3472    if (!res->obj->access_stage)
3473       imb.srcAccessMask = 0;
3474    if (res->obj->needs_zs_evaluate)
3475       imb.pNext = &res->obj->zs_evaluate;
3476    res->obj->needs_zs_evaluate = false;
3477    if (res->dmabuf_acquire) {
3478       imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
3479       imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
3480       res->dmabuf_acquire = false;
3481    }
3482    VkDependencyInfo dep = {
3483       VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3484       NULL,
3485       0,
3486       0,
3487       NULL,
3488       0,
3489       NULL,
3490       1,
3491       &imb
3492    };
3493    VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
3494
3495    resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
3496
3497    res->obj->access = imb.dstAccessMask;
3498    res->obj->access_stage = pipeline;
3499    res->layout = new_layout;
3500 }
3501
3502
3503 VkPipelineStageFlags
3504 zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)
3505 {
3506    switch (stage) {
3507    case VK_SHADER_STAGE_VERTEX_BIT:
3508       return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
3509    case VK_SHADER_STAGE_FRAGMENT_BIT:
3510       return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
3511    case VK_SHADER_STAGE_GEOMETRY_BIT:
3512       return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
3513    case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
3514       return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
3515    case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
3516       return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
3517    case VK_SHADER_STAGE_COMPUTE_BIT:
3518       return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
3519    default:
3520       unreachable("unknown shader stage bit");
3521    }
3522 }
3523
3524 ALWAYS_INLINE static VkPipelineStageFlags
3525 pipeline_access_stage(VkAccessFlags flags)
3526 {
3527    if (flags & (VK_ACCESS_UNIFORM_READ_BIT |
3528                 VK_ACCESS_SHADER_READ_BIT |
3529                 VK_ACCESS_SHADER_WRITE_BIT))
3530       return VK_PIPELINE_STAGE_TASK_SHADER_BIT_NV |
3531              VK_PIPELINE_STAGE_MESH_SHADER_BIT_NV |
3532              VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR |
3533              VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
3534              VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
3535              VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
3536              VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
3537              VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
3538              VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
3539    return VK_PIPELINE_STAGE_TRANSFER_BIT;
3540 }
3541
3542 ALWAYS_INLINE static bool
3543 zink_resource_buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3544 {
3545    if (!res->obj->access || !res->obj->access_stage)
3546       return true;
3547    if (!pipeline)
3548       pipeline = pipeline_access_stage(flags);
3549    return zink_resource_access_is_write(res->obj->access) ||
3550           zink_resource_access_is_write(flags) ||
3551           (res->obj->access_stage & pipeline) != pipeline ||
3552           (res->obj->access & flags) != flags;
3553 }
3554
3555 void
3556 zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3557 {
3558    VkMemoryBarrier bmb;
3559    if (!pipeline)
3560       pipeline = pipeline_access_stage(flags);
3561    if (!zink_resource_buffer_needs_barrier(res, flags, pipeline))
3562       return;
3563
3564    bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3565    bmb.pNext = NULL;
3566    bmb.srcAccessMask = res->obj->access;
3567    bmb.dstAccessMask = flags;
3568    if (!res->obj->access_stage)
3569       bmb.srcAccessMask = 0;
3570    bool is_write = zink_resource_access_is_write(flags);
3571    VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
3572    /* only barrier if we're changing layout or doing something besides read -> read */
3573    VKCTX(CmdPipelineBarrier)(
3574       cmdbuf,
3575       res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access),
3576       pipeline,
3577       0,
3578       1, &bmb,
3579       0, NULL,
3580       0, NULL
3581    );
3582
3583    resource_check_defer_buffer_barrier(ctx, res, pipeline);
3584
3585    res->obj->access = bmb.dstAccessMask;
3586    res->obj->access_stage = pipeline;
3587 }
3588
3589 void
3590 zink_resource_buffer_barrier2(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3591 {
3592    VkMemoryBarrier2 bmb;
3593    if (!pipeline)
3594       pipeline = pipeline_access_stage(flags);
3595    if (!zink_resource_buffer_needs_barrier(res, flags, pipeline))
3596       return;
3597    bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
3598    bmb.pNext = NULL;
3599    bmb.srcStageMask = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);
3600    bmb.srcAccessMask = res->obj->access;
3601    bmb.dstStageMask = pipeline;
3602    bmb.dstAccessMask = flags;
3603    if (!res->obj->access_stage)
3604       bmb.srcAccessMask = 0;
3605    bool is_write = zink_resource_access_is_write(flags);
3606    VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
3607    VkDependencyInfo dep = {
3608       VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3609       NULL,
3610       0,
3611       1,
3612       &bmb,
3613       0,
3614       NULL,
3615       0,
3616       NULL
3617    };
3618    /* only barrier if we're changing layout or doing something besides read -> read */
3619    VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
3620
3621    resource_check_defer_buffer_barrier(ctx, res, pipeline);
3622
3623    res->obj->access = bmb.dstAccessMask;
3624    res->obj->access_stage = pipeline;
3625 }
3626
3627 bool
3628 zink_resource_needs_barrier(struct zink_resource *res, VkImageLayout layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
3629 {
3630    if (res->base.b.target == PIPE_BUFFER)
3631       return zink_resource_buffer_needs_barrier(res, flags, pipeline);
3632    return zink_resource_image_needs_barrier(res, layout, flags, pipeline);
3633 }
3634
3635 static void
3636 zink_flush(struct pipe_context *pctx,
3637            struct pipe_fence_handle **pfence,
3638            unsigned flags)
3639 {
3640    struct zink_context *ctx = zink_context(pctx);
3641    bool deferred = flags & PIPE_FLUSH_DEFERRED;
3642    bool deferred_fence = false;
3643    struct zink_batch *batch = &ctx->batch;
3644    struct zink_fence *fence = NULL;
3645    struct zink_screen *screen = zink_screen(ctx->base.screen);
3646    unsigned submit_count = 0;
3647    VkSemaphore export_sem = VK_NULL_HANDLE;
3648
3649    /* triggering clears will force has_work */
3650    if (!deferred && ctx->clears_enabled) {
3651       /* if fbfetch outputs are active, disable them when flushing clears */
3652       unsigned fbfetch_outputs = ctx->fbfetch_outputs;
3653       if (fbfetch_outputs) {
3654          ctx->fbfetch_outputs = 0;
3655          ctx->rp_changed = true;
3656       }
3657       ctx->blitting = true;
3658       /* start rp to do all the clears */
3659       zink_batch_rp(ctx);
3660       ctx->blitting = false;
3661       ctx->fbfetch_outputs = fbfetch_outputs;
3662       ctx->rp_changed |= fbfetch_outputs > 0;
3663    }
3664
3665    if (ctx->needs_present && (flags & PIPE_FLUSH_END_OF_FRAME)) {
3666       if (ctx->needs_present->obj->image)
3667          zink_screen(ctx->base.screen)->image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
3668       ctx->needs_present = NULL;
3669    }
3670
3671    if (flags & PIPE_FLUSH_FENCE_FD) {
3672       assert(!deferred && pfence);
3673       const VkExportSemaphoreCreateInfo esci = {
3674          .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
3675          .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3676       };
3677       const VkSemaphoreCreateInfo sci = {
3678          .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3679          .pNext = &esci,
3680       };
3681       VkResult result = VKSCR(CreateSemaphore)(screen->dev, &sci, NULL, &export_sem);
3682       if (zink_screen_handle_vkresult(screen, result)) {
3683          assert(!batch->state->signal_semaphore);
3684          batch->state->signal_semaphore = export_sem;
3685          batch->has_work = true;
3686       } else {
3687          mesa_loge("ZINK: vkCreateSemaphore failed (%s)", vk_Result_to_str(result));
3688
3689          /* let flush proceed and ensure a null sem for fence_get_fd to return -1 */
3690          export_sem = VK_NULL_HANDLE;
3691       }
3692    }
3693
3694    if (!batch->has_work) {
3695        if (pfence) {
3696           /* reuse last fence */
3697           fence = ctx->last_fence;
3698        }
3699        if (!deferred) {
3700           struct zink_batch_state *last = zink_batch_state(ctx->last_fence);
3701           if (last) {
3702              sync_flush(ctx, last);
3703              if (last->is_device_lost)
3704                 check_device_lost(ctx);
3705           }
3706        }
3707        tc_driver_internal_flush_notify(ctx->tc);
3708    } else {
3709       fence = &batch->state->fence;
3710       submit_count = batch->state->submit_count;
3711       if (deferred && !(flags & PIPE_FLUSH_FENCE_FD) && pfence)
3712          deferred_fence = true;
3713       else
3714          flush_batch(ctx, true);
3715    }
3716
3717    if (pfence) {
3718       struct zink_tc_fence *mfence;
3719
3720       if (flags & TC_FLUSH_ASYNC) {
3721          mfence = zink_tc_fence(*pfence);
3722          assert(mfence);
3723       } else {
3724          mfence = zink_create_tc_fence();
3725
3726          screen->base.fence_reference(&screen->base, pfence, NULL);
3727          *pfence = (struct pipe_fence_handle *)mfence;
3728       }
3729
3730       mfence->fence = fence;
3731       mfence->sem = export_sem;
3732       if (fence)
3733          mfence->submit_count = submit_count;
3734
3735       if (deferred_fence) {
3736          assert(fence);
3737          mfence->deferred_ctx = pctx;
3738          assert(!ctx->deferred_fence || ctx->deferred_fence == fence);
3739          ctx->deferred_fence = fence;
3740       }
3741
3742       if (!fence || flags & TC_FLUSH_ASYNC) {
3743          if (!util_queue_fence_is_signalled(&mfence->ready))
3744             util_queue_fence_signal(&mfence->ready);
3745       }
3746    }
3747    if (fence) {
3748       if (!(flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC)))
3749          sync_flush(ctx, zink_batch_state(fence));
3750    }
3751    update_tc_info(ctx, false);
3752 }
3753
3754 void
3755 zink_fence_wait(struct pipe_context *pctx)
3756 {
3757    struct zink_context *ctx = zink_context(pctx);
3758
3759    if (ctx->batch.has_work)
3760       pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
3761    if (ctx->last_fence)
3762       stall(ctx);
3763 }
3764
3765 void
3766 zink_wait_on_batch(struct zink_context *ctx, uint64_t batch_id)
3767 {
3768    struct zink_batch_state *bs;
3769    if (!batch_id) {
3770       /* not submitted yet */
3771       flush_batch(ctx, true);
3772       bs = zink_batch_state(ctx->last_fence);
3773       assert(bs);
3774       batch_id = bs->fence.batch_id;
3775    }
3776    assert(batch_id);
3777    if (!zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, UINT64_MAX))
3778       check_device_lost(ctx);
3779 }
3780
3781 bool
3782 zink_check_batch_completion(struct zink_context *ctx, uint64_t batch_id)
3783 {
3784    assert(ctx->batch.state);
3785    if (!batch_id)
3786       /* not submitted yet */
3787       return false;
3788
3789    if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id))
3790       return true;
3791
3792    bool success = zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, 0);
3793    if (!success)
3794       check_device_lost(ctx);
3795    return success;
3796 }
3797
3798 static void
3799 zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
3800 {
3801    struct zink_context *ctx = zink_context(pctx);
3802    VkAccessFlags dst = flags == PIPE_TEXTURE_BARRIER_FRAMEBUFFER ?
3803                        VK_ACCESS_INPUT_ATTACHMENT_READ_BIT :
3804                        VK_ACCESS_SHADER_READ_BIT;
3805
3806    if (!ctx->framebuffer || !ctx->framebuffer->state.num_attachments)
3807       return;
3808
3809    /* if this is a fb barrier, flush all pending clears */
3810    if (ctx->rp_clears_enabled && dst == VK_ACCESS_INPUT_ATTACHMENT_READ_BIT)
3811       zink_batch_rp(ctx);
3812
3813    /* this is not an in-renderpass barrier */
3814    if (!ctx->fbfetch_outputs)
3815       zink_batch_no_rp(ctx);
3816
3817    if (zink_screen(ctx->base.screen)->info.have_KHR_synchronization2) {
3818       VkDependencyInfo dep = {0};
3819       dep.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
3820       dep.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
3821       dep.memoryBarrierCount = 1;
3822
3823       VkMemoryBarrier2 dmb = {0};
3824       dmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
3825       dmb.pNext = NULL;
3826       dmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3827       dmb.dstAccessMask = dst;
3828       dmb.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
3829       dmb.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT;
3830       dep.pMemoryBarriers = &dmb;
3831
3832       /* if zs fbfetch is a thing?
3833       if (ctx->fb_state.zsbuf) {
3834          const VkPipelineStageFlagBits2 depth_flags = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
3835          dmb.dstAccessMask |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
3836          dmb.srcStageMask |= depth_flags;
3837          dmb.dstStageMask |= depth_flags;
3838       }
3839       */
3840       VKCTX(CmdPipelineBarrier2)(ctx->batch.state->cmdbuf, &dep);
3841    } else {
3842       VkMemoryBarrier bmb = {0};
3843       bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3844       bmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3845       bmb.dstAccessMask = dst;
3846       VKCTX(CmdPipelineBarrier)(
3847          ctx->batch.state->cmdbuf,
3848          VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
3849          VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
3850          0,
3851          1, &bmb,
3852          0, NULL,
3853          0, NULL
3854       );
3855    }
3856 }
3857
3858 static inline void
3859 mem_barrier(struct zink_context *ctx, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlags src, VkAccessFlags dst)
3860 {
3861    struct zink_batch *batch = &ctx->batch;
3862    VkMemoryBarrier mb;
3863    mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3864    mb.pNext = NULL;
3865    mb.srcAccessMask = src;
3866    mb.dstAccessMask = dst;
3867    zink_batch_no_rp(ctx);
3868    VKCTX(CmdPipelineBarrier)(batch->state->cmdbuf, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL);
3869 }
3870
3871 void
3872 zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute)
3873 {
3874    const VkPipelineStageFlags gfx_flags = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
3875                                           VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
3876                                           VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
3877                                           VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
3878                                           VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
3879    const VkPipelineStageFlags cs_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
3880    VkPipelineStageFlags src = ctx->batch.last_was_compute ? cs_flags : gfx_flags;
3881    VkPipelineStageFlags dst = is_compute ? cs_flags : gfx_flags;
3882
3883    if (ctx->memory_barrier & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_SHADER_BUFFER | PIPE_BARRIER_IMAGE))
3884       mem_barrier(ctx, src, dst, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
3885
3886    if (ctx->memory_barrier & PIPE_BARRIER_CONSTANT_BUFFER)
3887       mem_barrier(ctx, src, dst,
3888                   VK_ACCESS_SHADER_WRITE_BIT,
3889                   VK_ACCESS_UNIFORM_READ_BIT);
3890
3891    if (!is_compute) {
3892       if (ctx->memory_barrier & PIPE_BARRIER_INDIRECT_BUFFER)
3893          mem_barrier(ctx, src, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
3894                      VK_ACCESS_SHADER_WRITE_BIT,
3895                      VK_ACCESS_INDIRECT_COMMAND_READ_BIT);
3896       if (ctx->memory_barrier & PIPE_BARRIER_VERTEX_BUFFER)
3897          mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
3898                      VK_ACCESS_SHADER_WRITE_BIT,
3899                      VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
3900
3901       if (ctx->memory_barrier & PIPE_BARRIER_INDEX_BUFFER)
3902          mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
3903                      VK_ACCESS_SHADER_WRITE_BIT,
3904                      VK_ACCESS_INDEX_READ_BIT);
3905       if (ctx->memory_barrier & PIPE_BARRIER_FRAMEBUFFER)
3906          zink_texture_barrier(&ctx->base, 0);
3907       if (ctx->memory_barrier & PIPE_BARRIER_STREAMOUT_BUFFER)
3908          mem_barrier(ctx, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
3909                             VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
3910                             VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT,
3911                      VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT,
3912                      VK_ACCESS_SHADER_READ_BIT,
3913                      VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT |
3914                      VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT);
3915    }
3916    ctx->memory_barrier = 0;
3917 }
3918
3919 static void
3920 zink_memory_barrier(struct pipe_context *pctx, unsigned flags)
3921 {
3922    struct zink_context *ctx = zink_context(pctx);
3923
3924    flags &= ~PIPE_BARRIER_UPDATE;
3925    if (!flags)
3926       return;
3927
3928    if (flags & PIPE_BARRIER_MAPPED_BUFFER) {
3929       /* TODO: this should flush all persistent buffers in use as I think */
3930       flags &= ~PIPE_BARRIER_MAPPED_BUFFER;
3931    }
3932    ctx->memory_barrier = flags;
3933 }
3934
3935 static void
3936 zink_flush_resource(struct pipe_context *pctx,
3937                     struct pipe_resource *pres)
3938 {
3939    struct zink_context *ctx = zink_context(pctx);
3940    struct zink_resource *res = zink_resource(pres);
3941    if (res->obj->dt) {
3942       if (zink_kopper_acquired(res->obj->dt, res->obj->dt_idx)) {
3943          zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
3944          zink_batch_reference_resource_rw(&ctx->batch, res, true);
3945       } else {
3946          ctx->needs_present = res;
3947       }
3948       ctx->batch.swapchain = res;
3949    } else if (res->dmabuf)
3950       res->dmabuf_acquire = true;
3951 }
3952
3953 static struct pipe_stream_output_target *
3954 zink_create_stream_output_target(struct pipe_context *pctx,
3955                                  struct pipe_resource *pres,
3956                                  unsigned buffer_offset,
3957                                  unsigned buffer_size)
3958 {
3959    struct zink_so_target *t;
3960    t = CALLOC_STRUCT(zink_so_target);
3961    if (!t)
3962       return NULL;
3963
3964    t->counter_buffer = pipe_buffer_create(pctx->screen, PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_DEFAULT, 4);
3965    if (!t->counter_buffer) {
3966       FREE(t);
3967       return NULL;
3968    }
3969
3970    t->base.reference.count = 1;
3971    t->base.context = pctx;
3972    pipe_resource_reference(&t->base.buffer, pres);
3973    t->base.buffer_offset = buffer_offset;
3974    t->base.buffer_size = buffer_size;
3975
3976    zink_resource(t->base.buffer)->so_valid = true;
3977
3978    return &t->base;
3979 }
3980
3981 static void
3982 zink_stream_output_target_destroy(struct pipe_context *pctx,
3983                                   struct pipe_stream_output_target *psot)
3984 {
3985    struct zink_so_target *t = (struct zink_so_target *)psot;
3986    pipe_resource_reference(&t->counter_buffer, NULL);
3987    pipe_resource_reference(&t->base.buffer, NULL);
3988    FREE(t);
3989 }
3990
3991 static void
3992 zink_set_stream_output_targets(struct pipe_context *pctx,
3993                                unsigned num_targets,
3994                                struct pipe_stream_output_target **targets,
3995                                const unsigned *offsets)
3996 {
3997    struct zink_context *ctx = zink_context(pctx);
3998
3999    /* always set counter_buffer_valid=false on unbind:
4000     * - on resume (indicated by offset==-1), set counter_buffer_valid=true
4001     * - otherwise the counter buffer is invalidated
4002     */
4003
4004    if (num_targets == 0) {
4005       for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4006          if (ctx->so_targets[i]) {
4007             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4008             if (so) {
4009                so->so_bind_count--;
4010                update_res_bind_count(ctx, so, false, true);
4011             }
4012          }
4013          pipe_so_target_reference(&ctx->so_targets[i], NULL);
4014       }
4015       ctx->num_so_targets = 0;
4016    } else {
4017       for (unsigned i = 0; i < num_targets; i++) {
4018          struct zink_so_target *t = zink_so_target(targets[i]);
4019          pipe_so_target_reference(&ctx->so_targets[i], targets[i]);
4020          if (!t)
4021             continue;
4022          if (offsets[0] != (unsigned)-1)
4023             t->counter_buffer_valid = false;
4024          struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4025          if (so) {
4026             so->so_bind_count++;
4027             update_res_bind_count(ctx, so, false, false);
4028          }
4029       }
4030       for (unsigned i = num_targets; i < ctx->num_so_targets; i++) {
4031          if (ctx->so_targets[i]) {
4032             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4033             if (so) {
4034                so->so_bind_count--;
4035                update_res_bind_count(ctx, so, false, true);
4036             }
4037          }
4038          pipe_so_target_reference(&ctx->so_targets[i], NULL);
4039       }
4040       ctx->num_so_targets = num_targets;
4041
4042       /* TODO: possibly avoid rebinding on resume if resuming from same buffers? */
4043       ctx->dirty_so_targets = true;
4044    }
4045 }
4046
4047 void
4048 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res)
4049 {
4050    if (!ctx->framebuffer)
4051       return;
4052    bool did_rebind = false;
4053    if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
4054       for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
4055          if (!ctx->fb_state.cbufs[i] ||
4056              zink_resource(ctx->fb_state.cbufs[i]->texture) != res)
4057             continue;
4058          zink_rebind_ctx_surface(ctx, &ctx->fb_state.cbufs[i]);
4059          did_rebind = true;
4060       }
4061    } else {
4062       if (ctx->fb_state.zsbuf && zink_resource(ctx->fb_state.zsbuf->texture) != res) {
4063          zink_rebind_ctx_surface(ctx, &ctx->fb_state.zsbuf);
4064          did_rebind = true;
4065       }
4066    }
4067
4068    did_rebind |= rebind_fb_state(ctx, res, false);
4069
4070    if (!did_rebind)
4071       return;
4072
4073    zink_batch_no_rp(ctx);
4074    struct zink_framebuffer *fb = zink_get_framebuffer(ctx);
4075    ctx->fb_changed |= ctx->framebuffer != fb;
4076    ctx->framebuffer = fb;
4077 }
4078
4079 ALWAYS_INLINE static struct zink_resource *
4080 rebind_ubo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4081 {
4082    struct zink_resource *res = update_descriptor_state_ubo(ctx, shader, slot,
4083                                                            ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]);
4084    zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, slot, 1);
4085    return res;
4086 }
4087
4088 ALWAYS_INLINE static struct zink_resource *
4089 rebind_ssbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4090 {
4091    const struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4092    struct zink_resource *res = zink_resource(ssbo->buffer);
4093    if (!res)
4094       return NULL;
4095    util_range_add(&res->base.b, &res->valid_buffer_range, ssbo->buffer_offset,
4096                   ssbo->buffer_offset + ssbo->buffer_size);
4097    update_descriptor_state_ssbo(ctx, shader, slot, res);
4098    zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SSBO, slot, 1);
4099    return res;
4100 }
4101
4102 ALWAYS_INLINE static struct zink_resource *
4103 rebind_tbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4104 {
4105    struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4106    if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER)
4107       return NULL;
4108    struct zink_resource *res = zink_resource(sampler_view->base.texture);
4109    VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci;
4110    bvci.buffer = res->obj->buffer;
4111    zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL);
4112    sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4113    update_descriptor_state_sampler(ctx, shader, slot, res);
4114    zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
4115    return res;
4116 }
4117
4118 ALWAYS_INLINE static struct zink_resource *
4119 rebind_ibo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4120 {
4121    struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4122    struct zink_resource *res = zink_resource(image_view->base.resource);
4123    if (!res || res->base.b.target != PIPE_BUFFER)
4124       return NULL;
4125    VkBufferViewCreateInfo bvci = image_view->buffer_view->bvci;
4126    bvci.buffer = res->obj->buffer;
4127    zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
4128    if (!zink_resource_object_init_storage(ctx, res)) {
4129       debug_printf("couldn't create storage image!");
4130       return NULL;
4131    }
4132    image_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4133    assert(image_view->buffer_view);
4134    util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset,
4135                   image_view->base.u.buf.offset + image_view->base.u.buf.size);
4136    update_descriptor_state_image(ctx, shader, slot, res);
4137    zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_IMAGE, slot, 1);
4138    return res;
4139 }
4140
4141 static unsigned
4142 rebind_buffer(struct zink_context *ctx, struct zink_resource *res, uint32_t rebind_mask, const unsigned expected_num_rebinds)
4143 {
4144    unsigned num_rebinds = 0;
4145    bool has_write = false;
4146
4147    if (!zink_resource_has_binds(res))
4148       return 0;
4149
4150    assert(!res->bindless[1]); //TODO
4151    if ((rebind_mask & BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER)) || (!rebind_mask && res->so_bind_count && ctx->num_so_targets)) {
4152       for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4153          if (ctx->so_targets[i]) {
4154             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4155             if (so && so == res) {
4156                ctx->dirty_so_targets = true;
4157                num_rebinds++;
4158             }
4159          }
4160       }
4161       rebind_mask &= ~BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER);
4162    }
4163    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4164       goto end;
4165
4166    if ((rebind_mask & BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER)) || (!rebind_mask && res->vbo_bind_mask)) {
4167       u_foreach_bit(slot, res->vbo_bind_mask) {
4168          if (ctx->vertex_buffers[slot].buffer.resource != &res->base.b) //wrong context
4169             goto end;
4170          num_rebinds++;
4171       }
4172       rebind_mask &= ~BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER);
4173       ctx->vertex_buffers_dirty = true;
4174    }
4175    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4176       goto end;
4177
4178    const uint32_t ubo_mask = rebind_mask ?
4179                              rebind_mask & BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES) :
4180                              ((res->ubo_bind_count[0] ? BITFIELD_RANGE(TC_BINDING_UBO_VS, (MESA_SHADER_STAGES - 1)) : 0) |
4181                               (res->ubo_bind_count[1] ? BITFIELD_BIT(TC_BINDING_UBO_CS) : 0));
4182    u_foreach_bit(shader, ubo_mask >> TC_BINDING_UBO_VS) {
4183       u_foreach_bit(slot, res->ubo_bind_mask[shader]) {
4184          if (&res->base.b != ctx->ubos[shader][slot].buffer) //wrong context
4185             goto end;
4186          rebind_ubo(ctx, shader, slot);
4187          num_rebinds++;
4188       }
4189    }
4190    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES);
4191    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4192       goto end;
4193
4194    const unsigned ssbo_mask = rebind_mask ?
4195                               rebind_mask & BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES) :
4196                               BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4197    u_foreach_bit(shader, ssbo_mask >> TC_BINDING_SSBO_VS) {
4198       u_foreach_bit(slot, res->ssbo_bind_mask[shader]) {
4199          struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4200          if (&res->base.b != ssbo->buffer) //wrong context
4201             goto end;
4202          rebind_ssbo(ctx, shader, slot);
4203          has_write |= (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0;
4204          num_rebinds++;
4205       }
4206    }
4207    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4208    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4209       goto end;
4210    const unsigned sampler_mask = rebind_mask ?
4211                                  rebind_mask & BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES) :
4212                                  BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4213    u_foreach_bit(shader, sampler_mask >> TC_BINDING_SAMPLERVIEW_VS) {
4214       u_foreach_bit(slot, res->sampler_binds[shader]) {
4215          struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4216          if (&res->base.b != sampler_view->base.texture) //wrong context
4217             goto end;
4218          rebind_tbo(ctx, shader, slot);
4219          num_rebinds++;
4220       }
4221    }
4222    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4223    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4224       goto end;
4225
4226    const unsigned image_mask = rebind_mask ?
4227                                rebind_mask & BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES) :
4228                                BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES);
4229    unsigned num_image_rebinds_remaining = rebind_mask ? expected_num_rebinds - num_rebinds : res->image_bind_count[0] + res->image_bind_count[1];
4230    u_foreach_bit(shader, image_mask >> TC_BINDING_IMAGE_VS) {
4231       for (unsigned slot = 0; num_image_rebinds_remaining && slot < ctx->di.num_images[shader]; slot++) {
4232          struct zink_resource *cres = ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_IMAGE][shader][slot];
4233          if (res != cres)
4234             continue;
4235
4236          rebind_ibo(ctx, shader, slot);
4237          const struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4238          has_write |= (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) != 0;
4239          num_image_rebinds_remaining--;
4240          num_rebinds++;
4241       }
4242    }
4243 end:
4244    if (num_rebinds)
4245       zink_batch_resource_usage_set(&ctx->batch, res, has_write, true);
4246    return num_rebinds;
4247 }
4248
4249 void
4250 zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4251                  unsigned dst_offset, unsigned src_offset, unsigned size)
4252 {
4253    VkBufferCopy region;
4254    region.srcOffset = src_offset;
4255    region.dstOffset = dst_offset;
4256    region.size = size;
4257
4258    struct zink_batch *batch = &ctx->batch;
4259    util_range_add(&dst->base.b, &dst->valid_buffer_range, dst_offset, dst_offset + size);
4260    zink_screen(ctx->base.screen)->buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, 0);
4261    zink_screen(ctx->base.screen)->buffer_barrier(ctx, dst, VK_ACCESS_TRANSFER_WRITE_BIT, 0);
4262    VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst);
4263    zink_batch_reference_resource_rw(batch, src, false);
4264    zink_batch_reference_resource_rw(batch, dst, true);
4265    VKCTX(CmdCopyBuffer)(cmdbuf, src->obj->buffer, dst->obj->buffer, 1, &region);
4266 }
4267
4268 void
4269 zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4270                        unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4271                        unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags)
4272 {
4273    struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst;
4274    struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src;
4275    struct zink_batch *batch = &ctx->batch;
4276    bool needs_present_readback = false;
4277
4278    bool buf2img = buf == src;
4279
4280    if (buf2img) {
4281       if (zink_is_swapchain(img)) {
4282          if (!zink_kopper_acquire(ctx, img, UINT64_MAX))
4283             return;
4284       }
4285       zink_screen(ctx->base.screen)->image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0, 0);
4286       zink_screen(ctx->base.screen)->buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
4287    } else {
4288       if (zink_is_swapchain(img))
4289          needs_present_readback = zink_kopper_acquire_readback(ctx, img);
4290       zink_screen(ctx->base.screen)->image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0);
4291       zink_screen(ctx->base.screen)->buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
4292       util_range_add(&dst->base.b, &dst->valid_buffer_range, dstx, dstx + src_box->width);
4293    }
4294
4295    VkBufferImageCopy region = {0};
4296    region.bufferOffset = buf2img ? src_box->x : dstx;
4297    region.bufferRowLength = 0;
4298    region.bufferImageHeight = 0;
4299    region.imageSubresource.mipLevel = buf2img ? dst_level : src_level;
4300    enum pipe_texture_target img_target = img->base.b.target;
4301    if (img->need_2D)
4302       img_target = img_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4303    switch (img_target) {
4304    case PIPE_TEXTURE_CUBE:
4305    case PIPE_TEXTURE_CUBE_ARRAY:
4306    case PIPE_TEXTURE_2D_ARRAY:
4307    case PIPE_TEXTURE_1D_ARRAY:
4308       /* these use layer */
4309       region.imageSubresource.baseArrayLayer = buf2img ? dstz : src_box->z;
4310       region.imageSubresource.layerCount = src_box->depth;
4311       region.imageOffset.z = 0;
4312       region.imageExtent.depth = 1;
4313       break;
4314    case PIPE_TEXTURE_3D:
4315       /* this uses depth */
4316       region.imageSubresource.baseArrayLayer = 0;
4317       region.imageSubresource.layerCount = 1;
4318       region.imageOffset.z = buf2img ? dstz : src_box->z;
4319       region.imageExtent.depth = src_box->depth;
4320       break;
4321    default:
4322       /* these must only copy one layer */
4323       region.imageSubresource.baseArrayLayer = 0;
4324       region.imageSubresource.layerCount = 1;
4325       region.imageOffset.z = 0;
4326       region.imageExtent.depth = 1;
4327    }
4328    region.imageOffset.x = buf2img ? dstx : src_box->x;
4329    region.imageOffset.y = buf2img ? dsty : src_box->y;
4330
4331    region.imageExtent.width = src_box->width;
4332    region.imageExtent.height = src_box->height;
4333
4334    /* never promote to unordered if swapchain was acquired */
4335    VkCommandBuffer cmdbuf = needs_present_readback ?
4336                             ctx->batch.state->cmdbuf :
4337                             buf2img ? zink_get_cmdbuf(ctx, buf, img) : zink_get_cmdbuf(ctx, img, buf);
4338    zink_batch_reference_resource_rw(batch, img, buf2img);
4339    zink_batch_reference_resource_rw(batch, buf, !buf2img);
4340
4341    /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage
4342     * to indicate whether to copy either the depth or stencil aspects
4343     */
4344    unsigned aspects = 0;
4345    if (map_flags) {
4346       assert((map_flags & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) !=
4347              (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY));
4348       if (map_flags & PIPE_MAP_DEPTH_ONLY)
4349          aspects = VK_IMAGE_ASPECT_DEPTH_BIT;
4350       else if (map_flags & PIPE_MAP_STENCIL_ONLY)
4351          aspects = VK_IMAGE_ASPECT_STENCIL_BIT;
4352    }
4353    if (!aspects)
4354       aspects = img->aspect;
4355    while (aspects) {
4356       int aspect = 1 << u_bit_scan(&aspects);
4357       region.imageSubresource.aspectMask = aspect;
4358
4359       /* this may or may not work with multisampled depth/stencil buffers depending on the driver implementation:
4360        *
4361        * srcImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4362        * - vkCmdCopyImageToBuffer spec
4363        *
4364        * dstImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4365        * - vkCmdCopyBufferToImage spec
4366        */
4367       if (buf2img)
4368          VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, img->obj->image, img->layout, 1, &region);
4369       else
4370          VKCTX(CmdCopyImageToBuffer)(cmdbuf, img->obj->image, img->layout, buf->obj->buffer, 1, &region);
4371    }
4372    if (needs_present_readback)
4373       zink_kopper_present_readback(ctx, img);
4374 }
4375
4376 static void
4377 zink_resource_copy_region(struct pipe_context *pctx,
4378                           struct pipe_resource *pdst,
4379                           unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4380                           struct pipe_resource *psrc,
4381                           unsigned src_level, const struct pipe_box *src_box)
4382 {
4383    struct zink_resource *dst = zink_resource(pdst);
4384    struct zink_resource *src = zink_resource(psrc);
4385    struct zink_context *ctx = zink_context(pctx);
4386    if (dst->base.b.target != PIPE_BUFFER && src->base.b.target != PIPE_BUFFER) {
4387       VkImageCopy region = {0};
4388       if (util_format_get_num_planes(src->base.b.format) == 1 &&
4389           util_format_get_num_planes(dst->base.b.format) == 1) {
4390       /* If neither the calling command’s srcImage nor the calling command’s dstImage
4391        * has a multi-planar image format then the aspectMask member of srcSubresource
4392        * and dstSubresource must match
4393        *
4394        * -VkImageCopy spec
4395        */
4396          assert(src->aspect == dst->aspect);
4397       } else
4398          unreachable("planar formats not yet handled");
4399
4400       zink_fb_clears_apply_or_discard(ctx, pdst, (struct u_rect){dstx, dstx + src_box->width, dsty, dsty + src_box->height}, false);
4401       zink_fb_clears_apply_region(ctx, psrc, zink_rect_from_box(src_box));
4402
4403       region.srcSubresource.aspectMask = src->aspect;
4404       region.srcSubresource.mipLevel = src_level;
4405       enum pipe_texture_target src_target = src->base.b.target;
4406       if (src->need_2D)
4407          src_target = src_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4408       switch (src_target) {
4409       case PIPE_TEXTURE_CUBE:
4410       case PIPE_TEXTURE_CUBE_ARRAY:
4411       case PIPE_TEXTURE_2D_ARRAY:
4412       case PIPE_TEXTURE_1D_ARRAY:
4413          /* these use layer */
4414          region.srcSubresource.baseArrayLayer = src_box->z;
4415          region.srcSubresource.layerCount = src_box->depth;
4416          region.srcOffset.z = 0;
4417          region.extent.depth = 1;
4418          break;
4419       case PIPE_TEXTURE_3D:
4420          /* this uses depth */
4421          region.srcSubresource.baseArrayLayer = 0;
4422          region.srcSubresource.layerCount = 1;
4423          region.srcOffset.z = src_box->z;
4424          region.extent.depth = src_box->depth;
4425          break;
4426       default:
4427          /* these must only copy one layer */
4428          region.srcSubresource.baseArrayLayer = 0;
4429          region.srcSubresource.layerCount = 1;
4430          region.srcOffset.z = 0;
4431          region.extent.depth = 1;
4432       }
4433
4434       region.srcOffset.x = src_box->x;
4435       region.srcOffset.y = src_box->y;
4436
4437       region.dstSubresource.aspectMask = dst->aspect;
4438       region.dstSubresource.mipLevel = dst_level;
4439       enum pipe_texture_target dst_target = dst->base.b.target;
4440       if (dst->need_2D)
4441          dst_target = dst_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4442       switch (dst_target) {
4443       case PIPE_TEXTURE_CUBE:
4444       case PIPE_TEXTURE_CUBE_ARRAY:
4445       case PIPE_TEXTURE_2D_ARRAY:
4446       case PIPE_TEXTURE_1D_ARRAY:
4447          /* these use layer */
4448          region.dstSubresource.baseArrayLayer = dstz;
4449          region.dstSubresource.layerCount = src_box->depth;
4450          region.dstOffset.z = 0;
4451          break;
4452       case PIPE_TEXTURE_3D:
4453          /* this uses depth */
4454          region.dstSubresource.baseArrayLayer = 0;
4455          region.dstSubresource.layerCount = 1;
4456          region.dstOffset.z = dstz;
4457          break;
4458       default:
4459          /* these must only copy one layer */
4460          region.dstSubresource.baseArrayLayer = 0;
4461          region.dstSubresource.layerCount = 1;
4462          region.dstOffset.z = 0;
4463       }
4464
4465       region.dstOffset.x = dstx;
4466       region.dstOffset.y = dsty;
4467       region.extent.width = src_box->width;
4468       region.extent.height = src_box->height;
4469
4470       struct zink_batch *batch = &ctx->batch;
4471       zink_resource_setup_transfer_layouts(ctx, src, dst);
4472       VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst);
4473       zink_batch_reference_resource_rw(batch, src, false);
4474       zink_batch_reference_resource_rw(batch, dst, true);
4475
4476       VKCTX(CmdCopyImage)(cmdbuf, src->obj->image, src->layout,
4477                      dst->obj->image, dst->layout,
4478                      1, &region);
4479    } else if (dst->base.b.target == PIPE_BUFFER &&
4480               src->base.b.target == PIPE_BUFFER) {
4481       zink_copy_buffer(ctx, dst, src, dstx, src_box->x, src_box->width);
4482    } else
4483       zink_copy_image_buffer(ctx, dst, src, dst_level, dstx, dsty, dstz, src_level, src_box, 0);
4484 }
4485
4486 static bool
4487 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
4488 {
4489    struct zink_context *ctx = zink_context(pctx);
4490    struct zink_resource *res = zink_resource(pres);
4491    struct zink_screen *screen = zink_screen(pctx->screen);
4492
4493    /* if any current usage exists, flush the queue */
4494    if (zink_resource_has_unflushed_usage(res))
4495       zink_flush_queue(ctx);
4496
4497    VkSemaphore sem = VK_NULL_HANDLE;
4498    bool ret = zink_bo_commit(screen, res, level, box, commit, &sem);
4499    if (ret) {
4500       if (sem)
4501          zink_batch_add_wait_semaphore(&ctx->batch, sem);
4502    } else {
4503       check_device_lost(ctx);
4504    }
4505
4506    return ret;
4507 }
4508
4509 static void
4510 rebind_image(struct zink_context *ctx, struct zink_resource *res)
4511 {
4512     zink_rebind_framebuffer(ctx, res);
4513     if (!zink_resource_has_binds(res))
4514        return;
4515     for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
4516        if (res->sampler_binds[i]) {
4517           for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
4518              struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
4519              if (sv && sv->base.texture == &res->base.b) {
4520                  struct pipe_surface *psurf = &sv->image_view->base;
4521                  zink_rebind_surface(ctx, &psurf);
4522                  sv->image_view = zink_surface(psurf);
4523                  zink_context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
4524                  update_descriptor_state_sampler(ctx, i, j, res);
4525              }
4526           }
4527        }
4528        if (!res->image_bind_count[i == MESA_SHADER_COMPUTE])
4529           continue;
4530        for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
4531           if (zink_resource(ctx->image_views[i][j].base.resource) == res) {
4532              zink_context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
4533              update_descriptor_state_image(ctx, i, j, res);
4534              _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
4535           }
4536        }
4537     }
4538 }
4539
4540 bool
4541 zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res)
4542 {
4543    if (res->base.b.target == PIPE_BUFFER) {
4544       /* force counter buffer reset */
4545       res->so_valid = false;
4546       return rebind_buffer(ctx, res, 0, 0) == res->bind_count[0] + res->bind_count[1];
4547    }
4548    rebind_image(ctx, res);
4549    return false;
4550 }
4551
4552 void
4553 zink_rebind_all_buffers(struct zink_context *ctx)
4554 {
4555    struct zink_batch *batch = &ctx->batch;
4556    ctx->vertex_buffers_dirty = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask > 0;
4557    ctx->dirty_so_targets = ctx->num_so_targets > 0;
4558    if (ctx->num_so_targets)
4559       zink_screen(ctx->base.screen)->buffer_barrier(ctx, zink_resource(ctx->dummy_xfb_buffer),
4560                                    VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
4561    for (unsigned shader = MESA_SHADER_VERTEX; shader < MESA_SHADER_STAGES; shader++) {
4562       for (unsigned slot = 0; slot < ctx->di.num_ubos[shader]; slot++) {
4563          struct zink_resource *res = rebind_ubo(ctx, shader, slot);
4564          if (res)
4565             zink_batch_resource_usage_set(batch, res, false, true);
4566       }
4567       for (unsigned slot = 0; slot < ctx->di.num_sampler_views[shader]; slot++) {
4568          struct zink_resource *res = rebind_tbo(ctx, shader, slot);
4569          if (res)
4570             zink_batch_resource_usage_set(batch, res, false, true);
4571       }
4572       for (unsigned slot = 0; slot < ctx->di.num_ssbos[shader]; slot++) {
4573          struct zink_resource *res = rebind_ssbo(ctx, shader, slot);
4574          if (res)
4575             zink_batch_resource_usage_set(batch, res, (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0, true);
4576       }
4577       for (unsigned slot = 0; slot < ctx->di.num_images[shader]; slot++) {
4578          struct zink_resource *res = rebind_ibo(ctx, shader, slot);
4579          if (res)
4580             zink_batch_resource_usage_set(batch, res, (ctx->image_views[shader][slot].base.access & PIPE_IMAGE_ACCESS_WRITE) != 0, true);
4581       }
4582    }
4583 }
4584
4585 void
4586 zink_rebind_all_images(struct zink_context *ctx)
4587 {
4588    rebind_fb_state(ctx, NULL, false);
4589     for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
4590       for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
4591          struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
4592          if (!sv)
4593             continue;
4594          struct zink_resource *res = zink_resource(sv->image_view->base.texture);
4595          if (res->obj != sv->image_view->obj) {
4596              struct pipe_surface *psurf = &sv->image_view->base;
4597              zink_rebind_surface(ctx, &psurf);
4598              sv->image_view = zink_surface(psurf);
4599              zink_context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
4600              update_descriptor_state_sampler(ctx, i, j, res);
4601          }
4602       }
4603       for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
4604          struct zink_image_view *image_view = &ctx->image_views[i][j];
4605          struct zink_resource *res = zink_resource(image_view->base.resource);
4606          if (!res)
4607             continue;
4608          if (ctx->image_views[i][j].surface->obj != res->obj) {
4609             zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
4610             image_view->surface = create_image_surface(ctx, &image_view->base, i == MESA_SHADER_COMPUTE);
4611             zink_context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
4612             update_descriptor_state_image(ctx, i, j, res);
4613             _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
4614          }
4615       }
4616    }
4617 }
4618
4619 static void
4620 zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *dst,
4621                                     struct pipe_resource *src, unsigned num_rebinds,
4622                                     uint32_t rebind_mask, uint32_t delete_buffer_id)
4623 {
4624    struct zink_resource *d = zink_resource(dst);
4625    struct zink_resource *s = zink_resource(src);
4626    struct zink_context *ctx = zink_context(pctx);
4627    struct zink_screen *screen = zink_screen(pctx->screen);
4628
4629    assert(d->internal_format == s->internal_format);
4630    assert(d->obj);
4631    assert(s->obj);
4632    util_idalloc_mt_free(&screen->buffer_ids, delete_buffer_id);
4633    /* add a ref just like check_resource_for_batch_ref() would've */
4634    if (zink_resource_has_binds(d) && zink_resource_has_usage(d))
4635       zink_batch_reference_resource(&ctx->batch, d);
4636    /* don't be too creative */
4637    zink_resource_object_reference(screen, &d->obj, s->obj);
4638    /* force counter buffer reset */
4639    d->so_valid = false;
4640    if (num_rebinds && rebind_buffer(ctx, d, rebind_mask, num_rebinds) < num_rebinds)
4641       ctx->buffer_rebind_counter = p_atomic_inc_return(&screen->buffer_rebind_counter);
4642 }
4643
4644 static bool
4645 zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage)
4646 {
4647    struct zink_screen *screen = zink_screen(pscreen);
4648    struct zink_resource *res = zink_resource(pres);
4649    uint32_t check_usage = 0;
4650    if (usage & PIPE_MAP_READ)
4651       check_usage |= ZINK_RESOURCE_ACCESS_WRITE;
4652    if (usage & PIPE_MAP_WRITE)
4653       check_usage |= ZINK_RESOURCE_ACCESS_RW;
4654    return !zink_resource_usage_check_completion(screen, res, check_usage);
4655 }
4656
4657 static void
4658 zink_emit_string_marker(struct pipe_context *pctx,
4659                         const char *string, int len)
4660 {
4661    struct zink_screen *screen = zink_screen(pctx->screen);
4662    struct zink_batch *batch = &zink_context(pctx)->batch;
4663
4664    /* make sure string is nul-terminated */
4665    char buf[512], *temp = NULL;
4666    if (len < ARRAY_SIZE(buf)) {
4667       memcpy(buf, string, len);
4668       buf[len] = '\0';
4669       string = buf;
4670    } else
4671       string = temp = strndup(string, len);
4672
4673    VkDebugUtilsLabelEXT label = {
4674       VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, NULL,
4675       string,
4676       { 0 }
4677    };
4678    screen->vk.CmdInsertDebugUtilsLabelEXT(batch->state->cmdbuf, &label);
4679    free(temp);
4680 }
4681
4682
4683 struct pipe_surface *
4684 zink_get_dummy_pipe_surface(struct zink_context *ctx, int samples_index)
4685 {
4686    if (!ctx->dummy_surface[samples_index]) {
4687       ctx->dummy_surface[samples_index] = zink_surface_create_null(ctx, PIPE_TEXTURE_2D, 1024, 1024, BITFIELD_BIT(samples_index));
4688       /* This is possibly used with imageLoad which according to GL spec must return 0 */
4689       if (!samples_index) {
4690          union pipe_color_union color = {0};
4691          struct pipe_box box;
4692          u_box_2d(0, 0, 1024, 1024, &box);
4693          ctx->base.clear_texture(&ctx->base, ctx->dummy_surface[samples_index]->texture, 0, &box, &color);
4694       }
4695    }
4696    return ctx->dummy_surface[samples_index];
4697 }
4698
4699 struct zink_surface *
4700 zink_get_dummy_surface(struct zink_context *ctx, int samples_index)
4701 {
4702    return zink_csurface(zink_get_dummy_pipe_surface(ctx, samples_index));
4703
4704 }
4705
4706 static void
4707 zink_tc_parse_dsa(void *state, struct tc_renderpass_info *info)
4708 {
4709    struct zink_depth_stencil_alpha_state *cso = state;
4710    info->zsbuf_write_dsa |= (cso->hw_state.depth_write || cso->base.stencil[0].writemask || cso->base.stencil[1].writemask);
4711    info->zsbuf_read_dsa |= (cso->hw_state.depth_test || cso->hw_state.stencil_test);
4712    /* TODO: if zsbuf fbfetch is ever supported */
4713 }
4714
4715 static void
4716 zink_tc_parse_fs(void *state, struct tc_renderpass_info *info)
4717 {
4718    struct zink_shader *zs = state;
4719    info->zsbuf_write_fs |= zs->nir->info.outputs_written & (BITFIELD64_BIT(FRAG_RESULT_DEPTH) | BITFIELD64_BIT(FRAG_RESULT_STENCIL));
4720    /* TODO: if >1 fbfetch attachment is ever supported */
4721    info->cbuf_fbfetch |= zs->nir->info.fs.uses_fbfetch_output ? BITFIELD_BIT(0) : 0;
4722 }
4723
4724 struct pipe_context *
4725 zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
4726 {
4727    struct zink_screen *screen = zink_screen(pscreen);
4728    struct zink_context *ctx = rzalloc(NULL, struct zink_context);
4729    bool is_copy_only = (flags & ZINK_CONTEXT_COPY_ONLY) > 0;
4730    bool is_compute_only = (flags & PIPE_CONTEXT_COMPUTE_ONLY) > 0;
4731    if (!ctx)
4732       goto fail;
4733
4734    ctx->flags = flags;
4735    ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
4736    ctx->gfx_pipeline_state.dirty = true;
4737    ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = 1;
4738    ctx->gfx_pipeline_state.uses_dynamic_stride = screen->info.have_EXT_extended_dynamic_state ||
4739                                                  screen->info.have_EXT_vertex_input_dynamic_state;
4740    ctx->compute_pipeline_state.dirty = true;
4741    ctx->fb_changed = ctx->rp_changed = true;
4742    ctx->sample_mask_changed = true;
4743    ctx->gfx_pipeline_state.gfx_prim_mode = PIPE_PRIM_MAX;
4744
4745    zink_init_draw_functions(ctx, screen);
4746    zink_init_grid_functions(ctx);
4747
4748    ctx->base.screen = pscreen;
4749    ctx->base.priv = priv;
4750
4751    ctx->base.destroy = zink_context_destroy;
4752    ctx->base.get_device_reset_status = zink_get_device_reset_status;
4753    ctx->base.set_device_reset_callback = zink_set_device_reset_callback;
4754
4755    zink_context_state_init(&ctx->base);
4756
4757    ctx->base.create_sampler_state = zink_create_sampler_state;
4758    ctx->base.bind_sampler_states = screen->info.have_EXT_non_seamless_cube_map ? zink_bind_sampler_states : zink_bind_sampler_states_nonseamless;
4759    ctx->base.delete_sampler_state = zink_delete_sampler_state;
4760
4761    ctx->base.create_sampler_view = zink_create_sampler_view;
4762    ctx->base.set_sampler_views = zink_set_sampler_views;
4763    ctx->base.sampler_view_destroy = zink_sampler_view_destroy;
4764    ctx->base.get_sample_position = zink_get_sample_position;
4765    ctx->base.set_sample_locations = zink_set_sample_locations;
4766
4767    zink_program_init(ctx);
4768
4769    ctx->base.set_polygon_stipple = zink_set_polygon_stipple;
4770    ctx->base.set_vertex_buffers = zink_set_vertex_buffers;
4771    ctx->base.set_viewport_states = zink_set_viewport_states;
4772    ctx->base.set_scissor_states = zink_set_scissor_states;
4773    ctx->base.set_inlinable_constants = zink_set_inlinable_constants;
4774    ctx->base.set_constant_buffer = zink_set_constant_buffer;
4775    ctx->base.set_shader_buffers = zink_set_shader_buffers;
4776    ctx->base.set_shader_images = zink_set_shader_images;
4777    ctx->base.set_framebuffer_state = zink_set_framebuffer_state;
4778    ctx->base.set_stencil_ref = zink_set_stencil_ref;
4779    ctx->base.set_clip_state = zink_set_clip_state;
4780    ctx->base.set_blend_color = zink_set_blend_color;
4781    ctx->base.set_tess_state = zink_set_tess_state;
4782    ctx->base.set_patch_vertices = zink_set_patch_vertices;
4783
4784    ctx->base.set_min_samples = zink_set_min_samples;
4785    ctx->gfx_pipeline_state.min_samples = 0;
4786    ctx->base.set_sample_mask = zink_set_sample_mask;
4787    ctx->gfx_pipeline_state.sample_mask = UINT32_MAX;
4788
4789    ctx->base.clear = zink_clear;
4790    ctx->base.clear_texture = zink_clear_texture;
4791    ctx->base.clear_buffer = zink_clear_buffer;
4792    ctx->base.clear_render_target = zink_clear_render_target;
4793    ctx->base.clear_depth_stencil = zink_clear_depth_stencil;
4794
4795    ctx->base.create_fence_fd = zink_create_fence_fd;
4796    ctx->base.fence_server_sync = zink_fence_server_sync;
4797    ctx->base.fence_server_signal = zink_fence_server_signal;
4798    ctx->base.flush = zink_flush;
4799    ctx->base.memory_barrier = zink_memory_barrier;
4800    ctx->base.texture_barrier = zink_texture_barrier;
4801    ctx->base.evaluate_depth_buffer = zink_evaluate_depth_buffer;
4802
4803    ctx->base.resource_commit = zink_resource_commit;
4804    ctx->base.resource_copy_region = zink_resource_copy_region;
4805    ctx->base.blit = zink_blit;
4806    ctx->base.create_stream_output_target = zink_create_stream_output_target;
4807    ctx->base.stream_output_target_destroy = zink_stream_output_target_destroy;
4808
4809    ctx->base.set_stream_output_targets = zink_set_stream_output_targets;
4810    ctx->base.flush_resource = zink_flush_resource;
4811
4812    ctx->base.emit_string_marker = zink_emit_string_marker;
4813
4814    zink_context_surface_init(&ctx->base);
4815    zink_context_resource_init(&ctx->base);
4816    zink_context_query_init(&ctx->base);
4817
4818    list_inithead(&ctx->query_pools);
4819    _mesa_set_init(&ctx->update_barriers[0][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4820    _mesa_set_init(&ctx->update_barriers[1][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4821    _mesa_set_init(&ctx->update_barriers[0][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4822    _mesa_set_init(&ctx->update_barriers[1][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4823    ctx->need_barriers[0] = &ctx->update_barriers[0][0];
4824    ctx->need_barriers[1] = &ctx->update_barriers[1][0];
4825
4826    slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
4827    slab_create_child(&ctx->transfer_pool_unsync, &screen->transfer_pool);
4828
4829    ctx->base.stream_uploader = u_upload_create_default(&ctx->base);
4830    ctx->base.const_uploader = u_upload_create_default(&ctx->base);
4831    for (int i = 0; i < ARRAY_SIZE(ctx->fb_clears); i++)
4832       util_dynarray_init(&ctx->fb_clears[i].clears, ctx);
4833
4834    if (!is_copy_only) {
4835       ctx->blitter = util_blitter_create(&ctx->base);
4836       if (!ctx->blitter)
4837          goto fail;
4838    }
4839
4840    zink_set_last_vertex_key(ctx)->last_vertex_stage = true;
4841    zink_set_tcs_key_patches(ctx, 1);
4842    if (!screen->optimal_keys) {
4843       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_VERTEX].size = sizeof(struct zink_vs_key_base);
4844       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_EVAL].size = sizeof(struct zink_vs_key_base);
4845       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_CTRL].size = sizeof(struct zink_tcs_key);
4846       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_GEOMETRY].size = sizeof(struct zink_vs_key_base);
4847       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].size = sizeof(struct zink_fs_key);
4848    }
4849    _mesa_hash_table_init(&ctx->framebuffer_cache, ctx, hash_framebuffer_imageless, equals_framebuffer_imageless);
4850    if (!zink_init_render_pass(ctx))
4851       goto fail;
4852    _mesa_set_init(&ctx->rendering_state_cache, ctx, hash_rendering_state, equals_rendering_state);
4853    ctx->dynamic_fb.info.pColorAttachments = ctx->dynamic_fb.attachments;
4854    ctx->dynamic_fb.info.sType = VK_STRUCTURE_TYPE_RENDERING_INFO;
4855    for (unsigned i = 0; i < ARRAY_SIZE(ctx->dynamic_fb.attachments); i++) {
4856       VkRenderingAttachmentInfo *att = &ctx->dynamic_fb.attachments[i];
4857       att->sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO;
4858       att->imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
4859       att->storeOp = VK_ATTACHMENT_STORE_OP_STORE;
4860    }
4861    ctx->gfx_pipeline_state.rendering_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO;
4862    ctx->gfx_pipeline_state.rendering_info.pColorAttachmentFormats = ctx->gfx_pipeline_state.rendering_formats;
4863    ctx->gfx_pipeline_state.feedback_loop = screen->driver_workarounds.always_feedback_loop;
4864
4865    const uint32_t data[] = {0};
4866    if (!is_copy_only) {
4867       ctx->dummy_vertex_buffer = pipe_buffer_create(&screen->base,
4868          PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_SHADER_IMAGE, PIPE_USAGE_IMMUTABLE, sizeof(data));
4869       if (!ctx->dummy_vertex_buffer)
4870          goto fail;
4871       ctx->dummy_xfb_buffer = pipe_buffer_create(&screen->base,
4872          PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_IMMUTABLE, sizeof(data));
4873       if (!ctx->dummy_xfb_buffer)
4874          goto fail;
4875    }
4876    if (!is_copy_only) {
4877       VkBufferViewCreateInfo bvci = create_bvci(ctx, zink_resource(ctx->dummy_vertex_buffer), PIPE_FORMAT_R8G8B8A8_UNORM, 0, sizeof(data));
4878       ctx->dummy_bufferview = get_buffer_view(ctx, zink_resource(ctx->dummy_vertex_buffer), &bvci);
4879       if (!ctx->dummy_bufferview)
4880          goto fail;
4881
4882       if (!zink_descriptors_init(ctx))
4883          goto fail;
4884    }
4885
4886    if (!is_copy_only && !is_compute_only) {
4887       ctx->base.create_texture_handle = zink_create_texture_handle;
4888       ctx->base.delete_texture_handle = zink_delete_texture_handle;
4889       ctx->base.make_texture_handle_resident = zink_make_texture_handle_resident;
4890       ctx->base.create_image_handle = zink_create_image_handle;
4891       ctx->base.delete_image_handle = zink_delete_image_handle;
4892       ctx->base.make_image_handle_resident = zink_make_image_handle_resident;
4893       for (unsigned i = 0; i < 2; i++) {
4894          _mesa_hash_table_init(&ctx->di.bindless[i].img_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4895          _mesa_hash_table_init(&ctx->di.bindless[i].tex_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4896
4897          /* allocate 1024 slots and reserve slot 0 */
4898          util_idalloc_init(&ctx->di.bindless[i].tex_slots, ZINK_MAX_BINDLESS_HANDLES);
4899          util_idalloc_alloc(&ctx->di.bindless[i].tex_slots);
4900          util_idalloc_init(&ctx->di.bindless[i].img_slots, ZINK_MAX_BINDLESS_HANDLES);
4901          util_idalloc_alloc(&ctx->di.bindless[i].img_slots);
4902          ctx->di.bindless[i].buffer_infos = malloc(sizeof(VkBufferView) * ZINK_MAX_BINDLESS_HANDLES);
4903          ctx->di.bindless[i].img_infos = malloc(sizeof(VkDescriptorImageInfo) * ZINK_MAX_BINDLESS_HANDLES);
4904          util_dynarray_init(&ctx->di.bindless[i].updates, NULL);
4905          util_dynarray_init(&ctx->di.bindless[i].resident, NULL);
4906       }
4907    }
4908
4909    zink_start_batch(ctx, &ctx->batch);
4910    if (!ctx->batch.state)
4911       goto fail;
4912
4913    if (!is_copy_only && !is_compute_only) {
4914       pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_vertex_buffer, 0, sizeof(data), data);
4915       pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_xfb_buffer, 0, sizeof(data), data);
4916       reapply_color_write(ctx);
4917
4918       /* set on startup just to avoid validation errors if a draw comes through without
4919       * a tess shader later
4920       */
4921       if (screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints)
4922          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, 1);
4923    }
4924    if (!is_copy_only) {
4925       for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
4926          /* need to update these based on screen config for null descriptors */
4927          for (unsigned j = 0; j < 32; j++) {
4928             update_descriptor_state_ubo(ctx, i, j, NULL);
4929             update_descriptor_state_sampler(ctx, i, j, NULL);
4930             update_descriptor_state_ssbo(ctx, i, j, NULL);
4931             update_descriptor_state_image(ctx, i, j, NULL);
4932          }
4933       }
4934       if (!screen->info.rb2_feats.nullDescriptor)
4935          ctx->di.fbfetch.imageView = zink_get_dummy_surface(ctx, 0)->image_view;
4936
4937       p_atomic_inc(&screen->base.num_contexts);
4938    }
4939
4940    zink_select_draw_vbo(ctx);
4941    zink_select_launch_grid(ctx);
4942
4943    if (!is_copy_only && zink_debug & ZINK_DEBUG_SHADERDB) {
4944       if (!screen->info.have_EXT_vertex_input_dynamic_state) {
4945          struct pipe_vertex_element velems[32] = {0};
4946          for (unsigned i = 0; i < ARRAY_SIZE(velems); i++)
4947             velems[i].src_format = PIPE_FORMAT_R8G8B8_UNORM;
4948          void *state = ctx->base.create_vertex_elements_state(&ctx->base, ARRAY_SIZE(velems), velems);
4949          ctx->base.bind_vertex_elements_state(&ctx->base, state);
4950       }
4951       ctx->gfx_pipeline_state.sample_mask = BITFIELD_MASK(32);
4952       struct pipe_framebuffer_state fb = {0};
4953       fb.cbufs[0] = zink_get_dummy_pipe_surface(ctx, 0);
4954       fb.nr_cbufs = 1;
4955       fb.width = fb.height = 256;
4956       ctx->base.set_framebuffer_state(&ctx->base, &fb);
4957       ctx->disable_color_writes = true;
4958       struct pipe_depth_stencil_alpha_state dsa = {0};
4959       void *state = ctx->base.create_depth_stencil_alpha_state(&ctx->base, &dsa);
4960       ctx->base.bind_depth_stencil_alpha_state(&ctx->base, state);
4961       zink_batch_rp(ctx);
4962    }
4963
4964    if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) {
4965       return &ctx->base;
4966    }
4967
4968    struct threaded_context *tc = (struct threaded_context*)threaded_context_create(&ctx->base, &screen->transfer_pool,
4969                                                      zink_context_replace_buffer_storage,
4970                                                      &(struct threaded_context_options){
4971                                                         .create_fence = zink_create_tc_fence_for_tc,
4972                                                         .is_resource_busy = zink_context_is_resource_busy,
4973                                                         .driver_calls_flush_notify = true,
4974                                                         .unsynchronized_get_device_reset_status = true,
4975                                                         .parse_renderpass_info = screen->driver_workarounds.track_renderpasses,
4976                                                         .dsa_parse = zink_tc_parse_dsa,
4977                                                         .fs_parse = zink_tc_parse_fs,
4978                                                      },
4979                                                      &ctx->tc);
4980
4981    if (tc && (struct zink_context*)tc != ctx) {
4982       threaded_context_init_bytes_mapped_limit(tc, 4);
4983       ctx->base.set_context_param = zink_set_context_param;
4984    }
4985
4986    return (struct pipe_context*)tc;
4987
4988 fail:
4989    if (ctx)
4990       zink_context_destroy(&ctx->base);
4991    return NULL;
4992 }
4993
4994 struct zink_context *
4995 zink_tc_context_unwrap(struct pipe_context *pctx, bool threaded)
4996 {
4997    /* need to get the actual zink_context, not the threaded context */
4998    if (threaded)
4999       pctx = threaded_context_unwrap_sync(pctx);
5000    pctx = trace_get_possibly_threaded_context(pctx);
5001    return zink_context(pctx);
5002 }