src/gallium/drivers/zink/zink_context.c

   1 /*
   2  * Copyright 2018 Collabora Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "zink_clear.h"
  25 #include "zink_context.h"
  26 #include "zink_descriptors.h"
  27 #include "zink_fence.h"
  28 #include "zink_format.h"
  29 #include "zink_framebuffer.h"
  30 #include "zink_helpers.h"
  31 #include "zink_inlines.h"
  32 #include "zink_kopper.h"
  33 #include "zink_pipeline.h"
  34 #include "zink_program.h"
  35 #include "zink_query.h"
  36 #include "zink_render_pass.h"
  37 #include "zink_resource.h"
  38 #include "zink_screen.h"
  39 #include "zink_state.h"
  40 #include "zink_surface.h"
  41
  42
  43 #include "util/u_blitter.h"
  44 #include "util/u_debug.h"
  45 #include "util/format_srgb.h"
  46 #include "util/format/u_format.h"
  47 #include "util/u_helpers.h"
  48 #include "util/u_inlines.h"
  49 #include "util/u_sample_positions.h"
  50 #include "util/u_string.h"
  51 #include "util/u_thread.h"
  52 #include "util/perf/u_trace.h"
  53 #include "util/u_cpu_detect.h"
  54 #include "util/strndup.h"
  55 #include "nir.h"
  56 #include "nir_builder.h"
  57
  58 #include "vk_format.h"
  59
  60 #include "driver_trace/tr_context.h"
  61
  62 #include "util/u_memory.h"
  63 #include "util/u_upload_mgr.h"
  64
  65 #define XXH_INLINE_ALL
  66 #include "util/xxhash.h"
  67
  68 static void
  69 update_tc_info(struct zink_context *ctx)
  70 {
  71    if (ctx->track_renderpasses) {
  72       const struct tc_renderpass_info *info = threaded_context_get_renderpass_info(ctx->tc);
  73       ctx->rp_changed |= ctx->dynamic_fb.tc_info.data != info->data;
  74       ctx->dynamic_fb.tc_info.data = info->data;
  75    } else {
  76       struct tc_renderpass_info info = ctx->dynamic_fb.tc_info;
  77       bool zsbuf_used = !ctx->zsbuf_unused;
  78       bool zsbuf_write = zink_is_zsbuf_write(ctx);
  79       ctx->dynamic_fb.tc_info.data32[0] = 0;
  80       if (ctx->clears_enabled & PIPE_CLEAR_DEPTHSTENCIL)
  81          ctx->dynamic_fb.tc_info.zsbuf_clear_partial = true;
  82       if (ctx->rp_clears_enabled & PIPE_CLEAR_DEPTHSTENCIL)
  83          ctx->dynamic_fb.tc_info.zsbuf_clear = true;
  84       if (ctx->dynamic_fb.tc_info.zsbuf_clear != info.zsbuf_clear)
  85          ctx->rp_loadop_changed = true;
  86       if (zink_is_zsbuf_write(ctx) != zsbuf_write)
  87          ctx->rp_layout_changed = true;
  88       ctx->rp_changed |= zink_is_zsbuf_used(ctx) != zsbuf_used;
  89    }
  90 }
  91
  92 void
  93 debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr)
  94 {
  95    sprintf(buf, "zink_buffer_view");
  96 }
  97
  98 ALWAYS_INLINE static void
  99 check_resource_for_batch_ref(struct zink_context *ctx, struct zink_resource *res)
 100 {
 101    if (!zink_resource_has_binds(res)) {
 102       /* avoid desync between usage and tracking:
 103        * - if usage exists, it must be removed before the context is destroyed
 104        * - having usage does not imply having tracking
 105        * - if tracking will be added here, also reapply usage to avoid dangling usage once tracking is removed
 106        * TODO: somehow fix this for perf because it's an extra hash lookup
 107        */
 108       if (!res->obj->dt && zink_resource_has_usage(res))
 109          zink_batch_reference_resource_rw(&ctx->batch, res, !!res->obj->bo->writes.u);
 110       else
 111          zink_batch_reference_resource(&ctx->batch, res);
 112    }
 113 }
 114
 115 static void
 116 zink_context_destroy(struct pipe_context *pctx)
 117 {
 118    struct zink_context *ctx = zink_context(pctx);
 119    struct zink_screen *screen = zink_screen(pctx->screen);
 120
 121    struct pipe_framebuffer_state fb = {0};
 122    pctx->set_framebuffer_state(pctx, &fb);
 123
 124    if (util_queue_is_initialized(&screen->flush_queue))
 125       util_queue_finish(&screen->flush_queue);
 126    if (ctx->batch.state && !screen->device_lost) {
 127       VkResult result = VKSCR(QueueWaitIdle)(screen->queue);
 128
 129       if (result != VK_SUCCESS)
 130          mesa_loge("ZINK: vkQueueWaitIdle failed (%s)", vk_Result_to_str(result));
 131    }
 132
 133    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) {
 134       simple_mtx_lock((&ctx->program_lock[i]));
 135       hash_table_foreach(&ctx->program_cache[i], entry) {
 136          struct zink_program *pg = entry->data;
 137          util_queue_fence_wait(&pg->cache_fence);
 138          pg->removed = true;
 139       }
 140       simple_mtx_unlock((&ctx->program_lock[i]));
 141    }
 142
 143    if (ctx->blitter)
 144       util_blitter_destroy(ctx->blitter);
 145    for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
 146       pipe_surface_release(&ctx->base, &ctx->fb_state.cbufs[i]);
 147    pipe_surface_release(&ctx->base, &ctx->fb_state.zsbuf);
 148
 149    pipe_resource_reference(&ctx->dummy_vertex_buffer, NULL);
 150    pipe_resource_reference(&ctx->dummy_xfb_buffer, NULL);
 151
 152    for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++)
 153       pipe_surface_release(&ctx->base, &ctx->dummy_surface[i]);
 154    zink_buffer_view_reference(screen, &ctx->dummy_bufferview, NULL);
 155
 156    zink_descriptors_deinit_bindless(ctx);
 157
 158    struct zink_batch_state *bs = ctx->batch_states;
 159    while (bs) {
 160       struct zink_batch_state *bs_next = bs->next;
 161       zink_clear_batch_state(ctx, bs);
 162       zink_batch_state_destroy(screen, bs);
 163       bs = bs_next;
 164    }
 165    bs = ctx->free_batch_states;
 166    while (bs) {
 167       struct zink_batch_state *bs_next = bs->next;
 168       zink_clear_batch_state(ctx, bs);
 169       zink_batch_state_destroy(screen, bs);
 170       bs = bs_next;
 171    }
 172    if (ctx->batch.state) {
 173       zink_clear_batch_state(ctx, ctx->batch.state);
 174       zink_batch_state_destroy(screen, ctx->batch.state);
 175    }
 176
 177    for (unsigned i = 0; i < 2; i++) {
 178       util_idalloc_fini(&ctx->di.bindless[i].tex_slots);
 179       util_idalloc_fini(&ctx->di.bindless[i].img_slots);
 180       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
 181          free(ctx->di.bindless[i].db.buffer_infos);
 182       else
 183          free(ctx->di.bindless[i].t.buffer_infos);
 184       free(ctx->di.bindless[i].img_infos);
 185       util_dynarray_fini(&ctx->di.bindless[i].updates);
 186       util_dynarray_fini(&ctx->di.bindless[i].resident);
 187    }
 188
 189    if (ctx->null_fs)
 190       pctx->delete_fs_state(pctx, ctx->null_fs);
 191
 192    hash_table_foreach(&ctx->framebuffer_cache, he)
 193       zink_destroy_framebuffer(screen, he->data);
 194
 195    hash_table_foreach(ctx->render_pass_cache, he)
 196       zink_destroy_render_pass(screen, he->data);
 197
 198    zink_context_destroy_query_pools(ctx);
 199    set_foreach(&ctx->gfx_inputs, he) {
 200       struct zink_gfx_input_key *ikey = (void*)he->key;
 201       VKSCR(DestroyPipeline)(screen->dev, ikey->pipeline, NULL);
 202    }
 203    set_foreach(&ctx->gfx_outputs, he) {
 204       struct zink_gfx_output_key *okey = (void*)he->key;
 205       VKSCR(DestroyPipeline)(screen->dev, okey->pipeline, NULL);
 206    }
 207    u_upload_destroy(pctx->stream_uploader);
 208    u_upload_destroy(pctx->const_uploader);
 209    slab_destroy_child(&ctx->transfer_pool);
 210    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++)
 211       _mesa_hash_table_clear(&ctx->program_cache[i], NULL);
 212    for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_lock); i++)
 213       simple_mtx_destroy(&ctx->program_lock[i]);
 214    _mesa_hash_table_destroy(ctx->render_pass_cache, NULL);
 215    slab_destroy_child(&ctx->transfer_pool_unsync);
 216
 217    if (zink_debug & ZINK_DEBUG_DGC) {
 218       for (unsigned i = 0; i < ARRAY_SIZE(ctx->dgc.upload); i++)
 219          u_upload_destroy(ctx->dgc.upload[i]);
 220       for (unsigned i = 0; i < ARRAY_SIZE(ctx->dgc.buffers); i++) {
 221          if (!ctx->dgc.buffers[i])
 222             continue;
 223          struct pipe_resource *pres = &ctx->dgc.buffers[i]->base.b;
 224          pipe_resource_reference(&pres, NULL);
 225       }
 226       util_dynarray_fini(&ctx->dgc.pipelines);
 227    }
 228
 229    zink_descriptors_deinit(ctx);
 230
 231    if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY))
 232       p_atomic_dec(&screen->base.num_contexts);
 233
 234    util_dynarray_foreach(&ctx->di.global_bindings, struct pipe_resource *, res) {
 235       pipe_resource_reference(res, NULL);
 236    }
 237    util_dynarray_fini(&ctx->di.global_bindings);
 238
 239    ralloc_free(ctx);
 240 }
 241
 242 static void
 243 check_device_lost(struct zink_context *ctx)
 244 {
 245    if (!zink_screen(ctx->base.screen)->device_lost || ctx->is_device_lost)
 246       return;
 247    debug_printf("ZINK: device lost detected!\n");
 248    if (ctx->reset.reset)
 249       ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
 250    ctx->is_device_lost = true;
 251 }
 252
 253 static enum pipe_reset_status
 254 zink_get_device_reset_status(struct pipe_context *pctx)
 255 {
 256    struct zink_context *ctx = zink_context(pctx);
 257
 258    enum pipe_reset_status status = PIPE_NO_RESET;
 259
 260    if (ctx->is_device_lost) {
 261       // Since we don't know what really happened to the hardware, just
 262       // assume that we are in the wrong
 263       status = PIPE_GUILTY_CONTEXT_RESET;
 264
 265       debug_printf("ZINK: device lost detected!\n");
 266
 267       if (ctx->reset.reset)
 268          ctx->reset.reset(ctx->reset.data, status);
 269    }
 270
 271    return status;
 272 }
 273
 274 static void
 275 zink_set_device_reset_callback(struct pipe_context *pctx,
 276                                const struct pipe_device_reset_callback *cb)
 277 {
 278    struct zink_context *ctx = zink_context(pctx);
 279    bool had_reset = !!ctx->reset.reset;
 280
 281    if (cb)
 282       ctx->reset = *cb;
 283    else
 284       memset(&ctx->reset, 0, sizeof(ctx->reset));
 285
 286    bool have_reset = !!ctx->reset.reset;
 287    if (had_reset != have_reset) {
 288       if (have_reset)
 289          p_atomic_inc(&zink_screen(pctx->screen)->robust_ctx_count);
 290       else
 291          p_atomic_dec(&zink_screen(pctx->screen)->robust_ctx_count);
 292    }
 293 }
 294
 295 static void
 296 zink_set_context_param(struct pipe_context *pctx, enum pipe_context_param param,
 297                        unsigned value)
 298 {
 299    struct zink_context *ctx = zink_context(pctx);
 300    struct zink_screen *screen = zink_screen(ctx->base.screen);
 301
 302    switch (param) {
 303    case PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE:
 304       if (screen->threaded_submit)
 305          util_set_thread_affinity(screen->flush_queue.threads[0],
 306                                  util_get_cpu_caps()->L3_affinity_mask[value],
 307                                  NULL, util_get_cpu_caps()->num_cpu_mask_bits);
 308       break;
 309    default:
 310       break;
 311    }
 312 }
 313
 314 static void
 315 zink_set_debug_callback(struct pipe_context *pctx, const struct util_debug_callback *cb)
 316 {
 317    struct zink_context *ctx = zink_context(pctx);
 318
 319    if (cb)
 320       ctx->dbg = *cb;
 321    else
 322       memset(&ctx->dbg, 0, sizeof(ctx->dbg));
 323 }
 324
 325 static VkSamplerMipmapMode
 326 sampler_mipmap_mode(enum pipe_tex_mipfilter filter)
 327 {
 328    switch (filter) {
 329    case PIPE_TEX_MIPFILTER_NEAREST: return VK_SAMPLER_MIPMAP_MODE_NEAREST;
 330    case PIPE_TEX_MIPFILTER_LINEAR: return VK_SAMPLER_MIPMAP_MODE_LINEAR;
 331    case PIPE_TEX_MIPFILTER_NONE:
 332       unreachable("PIPE_TEX_MIPFILTER_NONE should be dealt with earlier");
 333    }
 334    unreachable("unexpected filter");
 335 }
 336
 337 static VkSamplerAddressMode
 338 sampler_address_mode(enum pipe_tex_wrap filter)
 339 {
 340    switch (filter) {
 341    case PIPE_TEX_WRAP_REPEAT: return VK_SAMPLER_ADDRESS_MODE_REPEAT;
 342    case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
 343    case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
 344    case PIPE_TEX_WRAP_MIRROR_REPEAT: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
 345    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE;
 346    case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; /* not technically correct, but kinda works */
 347    default: break;
 348    }
 349    unreachable("unexpected wrap");
 350 }
 351
 352 static VkCompareOp
 353 compare_op(enum pipe_compare_func op)
 354 {
 355    switch (op) {
 356       case PIPE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
 357       case PIPE_FUNC_LESS: return VK_COMPARE_OP_LESS;
 358       case PIPE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
 359       case PIPE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
 360       case PIPE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
 361       case PIPE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
 362       case PIPE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
 363       case PIPE_FUNC_ALWAYS: return VK_COMPARE_OP_ALWAYS;
 364    }
 365    unreachable("unexpected compare");
 366 }
 367
 368 static inline bool
 369 wrap_needs_border_color(unsigned wrap)
 370 {
 371    return wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
 372           wrap == PIPE_TEX_WRAP_MIRROR_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER;
 373 }
 374
 375 static VkBorderColor
 376 get_border_color(const union pipe_color_union *color, bool is_integer, bool need_custom)
 377 {
 378    if (is_integer) {
 379       if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 0)
 380          return VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
 381       if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 1)
 382          return VK_BORDER_COLOR_INT_OPAQUE_BLACK;
 383       if (color->ui[0] == 1 && color->ui[1] == 1 && color->ui[2] == 1 && color->ui[3] == 1)
 384          return VK_BORDER_COLOR_INT_OPAQUE_WHITE;
 385       return need_custom ? VK_BORDER_COLOR_INT_CUSTOM_EXT : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
 386    }
 387
 388    if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 0)
 389       return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 390    if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 1)
 391       return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
 392    if (color->f[0] == 1 && color->f[1] == 1 && color->f[2] == 1 && color->f[3] == 1)
 393       return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
 394    return need_custom ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 395 }
 396
 397 static void *
 398 zink_create_sampler_state(struct pipe_context *pctx,
 399                           const struct pipe_sampler_state *state)
 400 {
 401    struct zink_screen *screen = zink_screen(pctx->screen);
 402    struct zink_context *zink = zink_context(pctx);
 403    bool need_custom = false;
 404    bool need_clamped_border_color = false;
 405    VkSamplerCreateInfo sci = {0};
 406    VkSamplerCustomBorderColorCreateInfoEXT cbci = {0};
 407    VkSamplerCustomBorderColorCreateInfoEXT cbci_clamped = {0};
 408    sci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
 409    if (screen->info.have_EXT_non_seamless_cube_map && !state->seamless_cube_map)
 410       sci.flags |= VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT;
 411    if (state->unnormalized_coords) {
 412       assert(zink->flags & PIPE_CONTEXT_COMPUTE_ONLY);
 413       sci.unnormalizedCoordinates = state->unnormalized_coords;
 414    }
 415    sci.magFilter = zink_filter(state->mag_img_filter);
 416    if (sci.unnormalizedCoordinates)
 417       sci.minFilter = sci.magFilter;
 418    else
 419       sci.minFilter = zink_filter(state->min_img_filter);
 420
 421    VkSamplerReductionModeCreateInfo rci;
 422    rci.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO;
 423    rci.pNext = NULL;
 424    switch (state->reduction_mode) {
 425    case PIPE_TEX_REDUCTION_MIN:
 426       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MIN;
 427       break;
 428    case PIPE_TEX_REDUCTION_MAX:
 429       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX;
 430       break;
 431    default:
 432       rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE;
 433       break;
 434    }
 435    if (state->reduction_mode)
 436       sci.pNext = &rci;
 437
 438    if (sci.unnormalizedCoordinates) {
 439       sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
 440    } else if (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE) {
 441       sci.mipmapMode = sampler_mipmap_mode(state->min_mip_filter);
 442       sci.minLod = state->min_lod;
 443       sci.maxLod = MAX2(state->max_lod, state->min_lod);
 444    } else {
 445       sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
 446       sci.minLod = 0;
 447       sci.maxLod = 0.25f;
 448    }
 449
 450    if (!sci.unnormalizedCoordinates) {
 451       sci.addressModeU = sampler_address_mode(state->wrap_s);
 452       sci.addressModeV = sampler_address_mode(state->wrap_t);
 453       sci.addressModeW = sampler_address_mode(state->wrap_r);
 454    } else {
 455       sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
 456    }
 457
 458    sci.mipLodBias = CLAMP(state->lod_bias,
 459                           -screen->info.props.limits.maxSamplerLodBias,
 460                           screen->info.props.limits.maxSamplerLodBias);
 461
 462    need_custom |= wrap_needs_border_color(state->wrap_s);
 463    need_custom |= wrap_needs_border_color(state->wrap_t);
 464    need_custom |= wrap_needs_border_color(state->wrap_r);
 465
 466    if (state->compare_mode == PIPE_TEX_COMPARE_NONE)
 467       sci.compareOp = VK_COMPARE_OP_NEVER;
 468    else {
 469       sci.compareOp = compare_op(state->compare_func);
 470       sci.compareEnable = VK_TRUE;
 471    }
 472
 473    bool is_integer = state->border_color_is_integer;
 474
 475    sci.borderColor = get_border_color(&state->border_color, is_integer, need_custom);
 476    if (sci.borderColor > VK_BORDER_COLOR_INT_OPAQUE_WHITE && need_custom) {
 477       if (!screen->info.border_color_feats.customBorderColorWithoutFormat &&
 478           screen->info.driver_props.driverID != VK_DRIVER_ID_MESA_TURNIP) {
 479          static bool warned = false;
 480          warn_missing_feature(warned, "customBorderColorWithoutFormat");
 481       }
 482       if (screen->info.have_EXT_custom_border_color &&
 483           (screen->info.border_color_feats.customBorderColorWithoutFormat || state->border_color_format)) {
 484          if (!screen->info.have_EXT_border_color_swizzle) {
 485             static bool warned = false;
 486             warn_missing_feature(warned, "VK_EXT_border_color_swizzle");
 487          }
 488
 489          if (!is_integer && !screen->have_D24_UNORM_S8_UINT) {
 490             union pipe_color_union clamped_border_color;
 491             for (unsigned i = 0; i < 4; ++i) {
 492                /* Use channel 0 on purpose, so that we can use OPAQUE_WHITE
 493                 * when the border color is 1.0. */
 494                clamped_border_color.f[i] = CLAMP(state->border_color.f[0], 0, 1);
 495             }
 496             if (memcmp(&state->border_color, &clamped_border_color, sizeof(clamped_border_color)) != 0) {
 497                need_clamped_border_color = true;
 498                cbci_clamped.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
 499                cbci_clamped.format = VK_FORMAT_UNDEFINED;
 500                /* these are identical unions */
 501                memcpy(&cbci_clamped.customBorderColor, &clamped_border_color, sizeof(union pipe_color_union));
 502             }
 503          }
 504          cbci.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
 505          if (screen->info.border_color_feats.customBorderColorWithoutFormat) {
 506             cbci.format = VK_FORMAT_UNDEFINED;
 507             /* these are identical unions */
 508             memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
 509          } else {
 510             if (util_format_is_depth_or_stencil(state->border_color_format)) {
 511                if (is_integer) {
 512                   cbci.format = VK_FORMAT_S8_UINT;
 513                   for (unsigned i = 0; i < 4; i++)
 514                      cbci.customBorderColor.uint32[i] = CLAMP(state->border_color.ui[i], 0, 255);
 515                } else {
 516                   cbci.format = zink_get_format(screen, util_format_get_depth_only(state->border_color_format));
 517                   /* these are identical unions */
 518                   memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
 519                }
 520             } else {
 521                cbci.format = zink_get_format(screen, state->border_color_format);
 522                union pipe_color_union color;
 523                for (unsigned i = 0; i < 4; i++) {
 524                   zink_format_clamp_channel_srgb(util_format_description(state->border_color_format), &color, &state->border_color, i);
 525                }
 526                zink_convert_color(screen, state->border_color_format, (void*)&cbci.customBorderColor, &color);
 527             }
 528          }
 529          cbci.pNext = sci.pNext;
 530          sci.pNext = &cbci;
 531          UNUSED uint32_t check = p_atomic_inc_return(&screen->cur_custom_border_color_samplers);
 532          assert(check <= screen->info.border_color_props.maxCustomBorderColorSamplers);
 533       } else
 534          sci.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; // TODO with custom shader if we're super interested?
 535       if (sci.unnormalizedCoordinates)
 536          sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
 537    }
 538
 539    if (state->max_anisotropy > 1) {
 540       sci.maxAnisotropy = state->max_anisotropy;
 541       sci.anisotropyEnable = VK_TRUE;
 542    }
 543
 544    struct zink_sampler_state *sampler = CALLOC_STRUCT(zink_sampler_state);
 545    if (!sampler)
 546       return NULL;
 547
 548    VkResult result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler);
 549    if (result != VK_SUCCESS) {
 550       mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
 551       FREE(sampler);
 552       return NULL;
 553    }
 554    if (need_clamped_border_color) {
 555       sci.pNext = &cbci_clamped;
 556       result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler_clamped);
 557       if (result != VK_SUCCESS) {
 558          mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result));
 559          VKSCR(DestroySampler)(screen->dev, sampler->sampler, NULL);
 560          FREE(sampler);
 561          return NULL;
 562       }
 563    }
 564    sampler->custom_border_color = need_custom;
 565    if (!screen->info.have_EXT_non_seamless_cube_map)
 566       sampler->emulate_nonseamless = !state->seamless_cube_map;
 567
 568    return sampler;
 569 }
 570
 571 ALWAYS_INLINE static VkImageLayout
 572 get_layout_for_binding(const struct zink_context *ctx, struct zink_resource *res, enum zink_descriptor_type type, bool is_compute)
 573 {
 574    if (res->obj->is_buffer)
 575       return 0;
 576    switch (type) {
 577    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
 578       return zink_descriptor_util_image_layout_eval(ctx, res, is_compute);
 579    case ZINK_DESCRIPTOR_TYPE_IMAGE:
 580       return VK_IMAGE_LAYOUT_GENERAL;
 581    default:
 582       break;
 583    }
 584    return 0;
 585 }
 586
 587 ALWAYS_INLINE static struct zink_surface *
 588 get_imageview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
 589 {
 590    switch (type) {
 591    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
 592       struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
 593       if (!sampler_view || !sampler_view->base.texture)
 594          return NULL;
 595       /* if this is a non-seamless cube sampler, return the cube array view */
 596       if (ctx->di.emulate_nonseamless[stage] & ctx->di.cubes[stage] & BITFIELD_BIT(idx))
 597          return sampler_view->cube_array;
 598       bool needs_zs_shader_swizzle = (ctx->di.zs_swizzle[stage].mask & BITFIELD_BIT(idx)) &&
 599                                      zink_screen(ctx->base.screen)->driver_workarounds.needs_zs_shader_swizzle;
 600       bool needs_shadow_shader_swizzle = (stage == MESA_SHADER_FRAGMENT) && ctx->gfx_stages[MESA_SHADER_FRAGMENT] &&
 601                                          (ctx->di.zs_swizzle[MESA_SHADER_FRAGMENT].mask & ctx->gfx_stages[MESA_SHADER_FRAGMENT]->fs.legacy_shadow_mask & BITFIELD_BIT(idx));
 602       if (sampler_view->zs_view && (needs_zs_shader_swizzle || needs_shadow_shader_swizzle))
 603          return sampler_view->zs_view;
 604       return sampler_view->image_view;
 605    }
 606    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
 607       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
 608       return image_view->base.resource ? image_view->surface : NULL;
 609    }
 610    default:
 611       break;
 612    }
 613    unreachable("ACK");
 614    return VK_NULL_HANDLE;
 615 }
 616
 617 ALWAYS_INLINE static struct zink_buffer_view *
 618 get_bufferview_for_binding(struct zink_context *ctx, gl_shader_stage stage, enum zink_descriptor_type type, unsigned idx)
 619 {
 620    switch (type) {
 621    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
 622       struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
 623       return sampler_view->base.texture ? sampler_view->buffer_view : NULL;
 624    }
 625    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
 626       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
 627       return image_view->base.resource ? image_view->buffer_view : NULL;
 628    }
 629    default:
 630       break;
 631    }
 632    unreachable("ACK");
 633    return VK_NULL_HANDLE;
 634 }
 635
 636 ALWAYS_INLINE static struct zink_resource *
 637 update_descriptor_state_ubo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 638 {
 639    struct zink_screen *screen = zink_screen(ctx->base.screen);
 640    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 641    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_UBO;
 642    ctx->di.descriptor_res[type][shader][slot] = res;
 643    if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 644       if (res)
 645          ctx->di.db.ubos[shader][slot].address = res->obj->bda + ctx->ubos[shader][slot].buffer_offset;
 646       else
 647          ctx->di.db.ubos[shader][slot].address = 0;
 648       ctx->di.db.ubos[shader][slot].range = res ? ctx->ubos[shader][slot].buffer_size : VK_WHOLE_SIZE;
 649       assert(ctx->di.db.ubos[shader][slot].range == VK_WHOLE_SIZE ||
 650              ctx->di.db.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
 651    } else {
 652       ctx->di.t.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset;
 653       if (res) {
 654          ctx->di.t.ubos[shader][slot].buffer = res->obj->buffer;
 655          ctx->di.t.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size;
 656          assert(ctx->di.t.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
 657       } else {
 658          VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
 659          ctx->di.t.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
 660          ctx->di.t.ubos[shader][slot].range = VK_WHOLE_SIZE;
 661       }
 662    }
 663    if (!slot) {
 664       if (res)
 665          ctx->di.push_valid |= BITFIELD64_BIT(shader);
 666       else
 667          ctx->di.push_valid &= ~BITFIELD64_BIT(shader);
 668    }
 669    return res;
 670 }
 671
 672 ALWAYS_INLINE static struct zink_resource *
 673 update_descriptor_state_ssbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 674 {
 675    struct zink_screen *screen = zink_screen(ctx->base.screen);
 676    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 677    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SSBO;
 678    ctx->di.descriptor_res[type][shader][slot] = res;
 679    if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 680       if (res)
 681          ctx->di.db.ssbos[shader][slot].address = res->obj->bda + ctx->ssbos[shader][slot].buffer_offset;
 682       else
 683          ctx->di.db.ssbos[shader][slot].address = 0;
 684       ctx->di.db.ssbos[shader][slot].range = res ? ctx->ssbos[shader][slot].buffer_size : VK_WHOLE_SIZE;
 685    } else {
 686       ctx->di.t.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset;
 687       if (res) {
 688          ctx->di.t.ssbos[shader][slot].buffer = res->obj->buffer;
 689          ctx->di.t.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size;
 690       } else {
 691          VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
 692          ctx->di.t.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
 693          ctx->di.t.ssbos[shader][slot].range = VK_WHOLE_SIZE;
 694       }
 695    }
 696    return res;
 697 }
 698
 699 ALWAYS_INLINE static struct zink_resource *
 700 update_descriptor_state_sampler(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 701 {
 702    struct zink_screen *screen = zink_screen(ctx->base.screen);
 703    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 704    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW;
 705    ctx->di.descriptor_res[type][shader][slot] = res;
 706    if (res) {
 707       if (res->obj->is_buffer) {
 708          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 709             ctx->di.db.tbos[shader][slot].address = res->obj->bda + ctx->sampler_views[shader][slot]->u.buf.offset;
 710             ctx->di.db.tbos[shader][slot].range = ctx->sampler_views[shader][slot]->u.buf.size;
 711             ctx->di.db.tbos[shader][slot].format = zink_get_format(screen, ctx->sampler_views[shader][slot]->format);
 712          } else {
 713             struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
 714             ctx->di.t.tbos[shader][slot] = bv->buffer_view;
 715          }
 716       } else {
 717          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
 718          ctx->di.textures[shader][slot].imageLayout = ctx->blitting ? res->layout : get_layout_for_binding(ctx, res, type, shader == MESA_SHADER_COMPUTE);
 719          ctx->di.textures[shader][slot].imageView = surface->image_view;
 720          if (!screen->have_D24_UNORM_S8_UINT &&
 721              ctx->sampler_states[shader][slot] && ctx->sampler_states[shader][slot]->sampler_clamped) {
 722             struct zink_sampler_state *state = ctx->sampler_states[shader][slot];
 723             VkSampler sampler = (surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
 724                                 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT) ?
 725                                 state->sampler_clamped :
 726                                 state->sampler;
 727             if (ctx->di.textures[shader][slot].sampler != sampler) {
 728                ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
 729                ctx->di.textures[shader][slot].sampler = sampler;
 730             }
 731          }
 732       }
 733    } else {
 734       if (likely(have_null_descriptors)) {
 735          ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE;
 736          ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
 737          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 738             ctx->di.db.tbos[shader][slot].address = 0;
 739             ctx->di.db.tbos[shader][slot].range = VK_WHOLE_SIZE;
 740          } else {
 741             ctx->di.t.tbos[shader][slot] = VK_NULL_HANDLE;
 742          }
 743       } else {
 744          assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB);
 745          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
 746          struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
 747          ctx->di.textures[shader][slot].imageView = null_surface->image_view;
 748          ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 749          ctx->di.t.tbos[shader][slot] = null_bufferview->buffer_view;
 750       }
 751    }
 752    return res;
 753 }
 754
 755 void
 756 zink_update_shadow_samplerviews(struct zink_context *ctx, unsigned mask)
 757 {
 758    u_foreach_bit(slot, mask)
 759       update_descriptor_state_sampler(ctx, MESA_SHADER_FRAGMENT, slot, ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][MESA_SHADER_FRAGMENT][slot]);
 760 }
 761
 762 ALWAYS_INLINE static struct zink_resource *
 763 update_descriptor_state_image(struct zink_context *ctx, gl_shader_stage shader, unsigned slot, struct zink_resource *res)
 764 {
 765    struct zink_screen *screen = zink_screen(ctx->base.screen);
 766    bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
 767    const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_IMAGE;
 768    ctx->di.descriptor_res[type][shader][slot] = res;
 769    if (res) {
 770       if (res->obj->is_buffer) {
 771          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 772             ctx->di.db.texel_images[shader][slot].address = res->obj->bda + ctx->image_views[shader][slot].base.u.buf.offset;
 773             ctx->di.db.texel_images[shader][slot].range = ctx->image_views[shader][slot].base.u.buf.size;
 774             ctx->di.db.texel_images[shader][slot].format = zink_get_format(screen, ctx->image_views[shader][slot].base.format);
 775          } else {
 776             struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
 777             ctx->di.t.texel_images[shader][slot] = bv->buffer_view;
 778          }
 779       } else {
 780          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
 781          ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
 782          ctx->di.images[shader][slot].imageView = surface->image_view;
 783       }
 784    } else {
 785       if (likely(have_null_descriptors)) {
 786          memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot]));
 787          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
 788             ctx->di.db.texel_images[shader][slot].address = 0;
 789             ctx->di.db.texel_images[shader][slot].range = VK_WHOLE_SIZE;
 790          } else {
 791             ctx->di.t.texel_images[shader][slot] = VK_NULL_HANDLE;
 792          }
 793       } else {
 794          assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB);
 795          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
 796          struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
 797          ctx->di.images[shader][slot].imageView = null_surface->image_view;
 798          ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
 799          ctx->di.t.texel_images[shader][slot] = null_bufferview->buffer_view;
 800       }
 801    }
 802    return res;
 803 }
 804
 805 static void
 806 update_nonseamless_shader_key(struct zink_context *ctx, gl_shader_stage pstage)
 807 {
 808    const uint32_t new_mask = ctx->di.emulate_nonseamless[pstage] & ctx->di.cubes[pstage];
 809    if (pstage == MESA_SHADER_COMPUTE) {
 810       if (ctx->compute_pipeline_state.key.base.nonseamless_cube_mask != new_mask)
 811          ctx->compute_dirty = true;
 812       ctx->compute_pipeline_state.key.base.nonseamless_cube_mask = new_mask;
 813    } else {
 814       if (zink_get_shader_key_base(ctx, pstage)->nonseamless_cube_mask != new_mask)
 815          zink_set_shader_key_base(ctx, pstage)->nonseamless_cube_mask = new_mask;
 816    }
 817 }
 818
 819 static void
 820 zink_bind_sampler_states(struct pipe_context *pctx,
 821                          gl_shader_stage shader,
 822                          unsigned start_slot,
 823                          unsigned num_samplers,
 824                          void **samplers)
 825 {
 826    struct zink_context *ctx = zink_context(pctx);
 827    struct zink_screen *screen = zink_screen(pctx->screen);
 828    for (unsigned i = 0; i < num_samplers; ++i) {
 829       struct zink_sampler_state *state = samplers[i];
 830       if (samplers[i] == ctx->sampler_states[shader][start_slot + i])
 831          continue;
 832       ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, 1);
 833       ctx->sampler_states[shader][start_slot + i] = state;
 834       if (state) {
 835          ctx->di.textures[shader][start_slot + i].sampler = state->sampler;
 836          if (state->sampler_clamped && !screen->have_D24_UNORM_S8_UINT) {
 837             struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
 838             if (surface &&
 839                 ((surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) ||
 840                  (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT)))
 841                ctx->di.textures[shader][start_slot + i].sampler = state->sampler_clamped;
 842          }
 843       } else {
 844          ctx->di.textures[shader][start_slot + i].sampler = VK_NULL_HANDLE;
 845       }
 846    }
 847    ctx->di.num_samplers[shader] = start_slot + num_samplers;
 848 }
 849
 850 static void
 851 zink_bind_sampler_states_nonseamless(struct pipe_context *pctx,
 852                                      gl_shader_stage shader,
 853                                      unsigned start_slot,
 854                                      unsigned num_samplers,
 855                                      void **samplers)
 856 {
 857    struct zink_context *ctx = zink_context(pctx);
 858    uint32_t old_mask = ctx->di.emulate_nonseamless[shader];
 859    uint32_t mask = BITFIELD_RANGE(start_slot, num_samplers);
 860    ctx->di.emulate_nonseamless[shader] &= ~mask;
 861    for (unsigned i = 0; i < num_samplers; ++i) {
 862       struct zink_sampler_state *state = samplers[i];
 863       const uint32_t bit = BITFIELD_BIT(start_slot + i);
 864       if (!state)
 865          continue;
 866       if (state->emulate_nonseamless)
 867          ctx->di.emulate_nonseamless[shader] |= bit;
 868       if (state->emulate_nonseamless != (old_mask & bit) && (ctx->di.cubes[shader] & bit)) {
 869          struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i);
 870          if (surface && ctx->di.images[shader][start_slot + i].imageView != surface->image_view) {
 871             ctx->di.images[shader][start_slot + i].imageView = surface->image_view;
 872             update_descriptor_state_sampler(ctx, shader, start_slot + i, zink_resource(surface->base.texture));
 873             ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i, 1);
 874          }
 875       }
 876    }
 877    zink_bind_sampler_states(pctx, shader, start_slot, num_samplers, samplers);
 878    update_nonseamless_shader_key(ctx, shader);
 879 }
 880
 881 static void
 882 zink_delete_sampler_state(struct pipe_context *pctx,
 883                           void *sampler_state)
 884 {
 885    struct zink_sampler_state *sampler = sampler_state;
 886    struct zink_batch *batch = &zink_context(pctx)->batch;
 887    /* may be called if context_create fails */
 888    if (batch->state) {
 889       util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
 890                            sampler->sampler);
 891       if (sampler->sampler_clamped)
 892          util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
 893                               sampler->sampler_clamped);
 894    }
 895    if (sampler->custom_border_color)
 896       p_atomic_dec(&zink_screen(pctx->screen)->cur_custom_border_color_samplers);
 897    FREE(sampler);
 898 }
 899
 900 static VkImageAspectFlags
 901 sampler_aspect_from_format(enum pipe_format fmt)
 902 {
 903    if (util_format_is_depth_or_stencil(fmt)) {
 904       const struct util_format_description *desc = util_format_description(fmt);
 905       if (util_format_has_depth(desc))
 906          return VK_IMAGE_ASPECT_DEPTH_BIT;
 907       assert(util_format_has_stencil(desc));
 908       return VK_IMAGE_ASPECT_STENCIL_BIT;
 909    } else
 910      return VK_IMAGE_ASPECT_COLOR_BIT;
 911 }
 912
 913 static uint32_t
 914 hash_bufferview(void *bvci)
 915 {
 916    size_t offset = offsetof(VkBufferViewCreateInfo, flags);
 917    return _mesa_hash_data((char*)bvci + offset, sizeof(VkBufferViewCreateInfo) - offset);
 918 }
 919
 920 static VkBufferViewCreateInfo
 921 create_bvci(struct zink_context *ctx, struct zink_resource *res, enum pipe_format format, uint32_t offset, uint32_t range)
 922 {
 923    struct zink_screen *screen = zink_screen(ctx->base.screen);
 924    VkBufferViewCreateInfo bvci;
 925    // Zero whole struct (including alignment holes), so hash_bufferview
 926    // does not access potentially uninitialized data.
 927    memset(&bvci, 0, sizeof(bvci));
 928    bvci.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO;
 929    bvci.pNext = NULL;
 930    if (screen->format_props[format].bufferFeatures & VK_FORMAT_FEATURE_STORAGE_TEXEL_BUFFER_BIT)
 931       bvci.buffer = res->obj->storage_buffer ? res->obj->storage_buffer : res->obj->buffer;
 932    else
 933       bvci.buffer = res->obj->buffer;
 934    bvci.format = zink_get_format(screen, format);
 935    assert(bvci.format);
 936    bvci.offset = offset;
 937    bvci.range = !offset && range == res->base.b.width0 ? VK_WHOLE_SIZE : range;
 938    unsigned blocksize = util_format_get_blocksize(format);
 939    if (bvci.range != VK_WHOLE_SIZE) {
 940       /* clamp out partial texels */
 941       bvci.range -= bvci.range % blocksize;
 942       if (bvci.offset + bvci.range >= res->base.b.width0)
 943          bvci.range = VK_WHOLE_SIZE;
 944    }
 945    uint64_t clamp = blocksize * screen->info.props.limits.maxTexelBufferElements;
 946    if (bvci.range == VK_WHOLE_SIZE && res->base.b.width0 > clamp)
 947       bvci.range = clamp;
 948    bvci.flags = 0;
 949    return bvci;
 950 }
 951
 952 static struct zink_buffer_view *
 953 get_buffer_view(struct zink_context *ctx, struct zink_resource *res, VkBufferViewCreateInfo *bvci)
 954 {
 955    struct zink_screen *screen = zink_screen(ctx->base.screen);
 956    struct zink_buffer_view *buffer_view = NULL;
 957
 958    uint32_t hash = hash_bufferview(bvci);
 959    simple_mtx_lock(&res->bufferview_mtx);
 960    struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, hash, bvci);
 961    if (he) {
 962       buffer_view = he->data;
 963       p_atomic_inc(&buffer_view->reference.count);
 964    } else {
 965       VkBufferView view;
 966       VkResult result = VKSCR(CreateBufferView)(screen->dev, bvci, NULL, &view);
 967       if (result != VK_SUCCESS) {
 968          mesa_loge("ZINK: vkCreateBufferView failed (%s)", vk_Result_to_str(result));
 969          goto out;
 970       }
 971       buffer_view = CALLOC_STRUCT(zink_buffer_view);
 972       if (!buffer_view) {
 973          VKSCR(DestroyBufferView)(screen->dev, view, NULL);
 974          goto out;
 975       }
 976       pipe_reference_init(&buffer_view->reference, 1);
 977       pipe_resource_reference(&buffer_view->pres, &res->base.b);
 978       buffer_view->bvci = *bvci;
 979       buffer_view->buffer_view = view;
 980       buffer_view->hash = hash;
 981       _mesa_hash_table_insert_pre_hashed(&res->bufferview_cache, hash, &buffer_view->bvci, buffer_view);
 982    }
 983 out:
 984    simple_mtx_unlock(&res->bufferview_mtx);
 985    return buffer_view;
 986 }
 987
 988 enum pipe_swizzle
 989 zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle)
 990 {
 991    switch (swizzle) {
 992    case PIPE_SWIZZLE_X:
 993    case PIPE_SWIZZLE_Y:
 994    case PIPE_SWIZZLE_Z:
 995    case PIPE_SWIZZLE_W:
 996       return desc->channel[swizzle].type == UTIL_FORMAT_TYPE_VOID ? PIPE_SWIZZLE_1 : swizzle;
 997    default:
 998       break;
 999    }
1000    return swizzle;
1001 }
1002
1003 ALWAYS_INLINE static enum pipe_swizzle
1004 clamp_zs_swizzle(enum pipe_swizzle swizzle)
1005 {
1006    switch (swizzle) {
1007    case PIPE_SWIZZLE_X:
1008    case PIPE_SWIZZLE_Y:
1009    case PIPE_SWIZZLE_Z:
1010    case PIPE_SWIZZLE_W:
1011       return PIPE_SWIZZLE_X;
1012    default:
1013       break;
1014    }
1015    return swizzle;
1016 }
1017
1018 ALWAYS_INLINE static enum pipe_swizzle
1019 clamp_alpha_swizzle(enum pipe_swizzle swizzle)
1020 {
1021    if (swizzle == PIPE_SWIZZLE_W)
1022       return PIPE_SWIZZLE_X;
1023    if (swizzle < PIPE_SWIZZLE_W)
1024       return PIPE_SWIZZLE_0;
1025    return swizzle;
1026 }
1027
1028 ALWAYS_INLINE static enum pipe_swizzle
1029 clamp_luminance_swizzle(enum pipe_swizzle swizzle)
1030 {
1031    if (swizzle == PIPE_SWIZZLE_W)
1032       return PIPE_SWIZZLE_1;
1033    if (swizzle < PIPE_SWIZZLE_W)
1034       return PIPE_SWIZZLE_X;
1035    return swizzle;
1036 }
1037
1038 ALWAYS_INLINE static enum pipe_swizzle
1039 clamp_luminance_alpha_swizzle(enum pipe_swizzle swizzle)
1040 {
1041    if (swizzle == PIPE_SWIZZLE_W)
1042       return PIPE_SWIZZLE_Y;
1043    if (swizzle < PIPE_SWIZZLE_W)
1044       return PIPE_SWIZZLE_X;
1045    return swizzle;
1046 }
1047
1048 ALWAYS_INLINE static bool
1049 viewtype_is_cube(const VkImageViewCreateInfo *ivci)
1050 {
1051    return ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE ||
1052           ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY;
1053 }
1054
1055 static struct pipe_sampler_view *
1056 zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres,
1057                          const struct pipe_sampler_view *state)
1058 {
1059    struct zink_screen *screen = zink_screen(pctx->screen);
1060    struct zink_resource *res = zink_resource(pres);
1061    struct zink_context *ctx = zink_context(pctx);
1062    struct zink_sampler_view *sampler_view = CALLOC_STRUCT_CL(zink_sampler_view);
1063    bool err;
1064
1065    if (!sampler_view) {
1066       mesa_loge("ZINK: failed to allocate sampler_view!");
1067       return NULL;
1068    }
1069
1070    sampler_view->base = *state;
1071    sampler_view->base.texture = NULL;
1072    pipe_resource_reference(&sampler_view->base.texture, pres);
1073    sampler_view->base.reference.count = 1;
1074    sampler_view->base.context = pctx;
1075
1076    if (state->target != PIPE_BUFFER) {
1077       VkImageViewCreateInfo ivci;
1078
1079       struct pipe_surface templ = {0};
1080       templ.u.tex.level = state->u.tex.first_level;
1081       templ.format = state->format;
1082       /* avoid needing mutable for depth/stencil sampling */
1083       if (util_format_is_depth_and_stencil(pres->format))
1084          templ.format = pres->format;
1085       if (state->target != PIPE_TEXTURE_3D) {
1086          templ.u.tex.first_layer = state->u.tex.first_layer;
1087          templ.u.tex.last_layer = state->u.tex.last_layer;
1088       }
1089
1090       if (zink_is_swapchain(res)) {
1091          if (!zink_kopper_acquire(ctx, res, UINT64_MAX)) {
1092             FREE_CL(sampler_view);
1093             return NULL;
1094          }
1095       }
1096
1097       ivci = create_ivci(screen, res, &templ, state->target);
1098       ivci.subresourceRange.levelCount = state->u.tex.last_level - state->u.tex.first_level + 1;
1099       ivci.subresourceRange.aspectMask = sampler_aspect_from_format(state->format);
1100       bool red_depth_sampler_view = false;
1101       /* samplers for stencil aspects of packed formats need to always use stencil swizzle */
1102       if (ivci.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
1103          ivci.components.r = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_r));
1104          ivci.components.g = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_g));
1105          ivci.components.b = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_b));
1106          ivci.components.a = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_a));
1107
1108          /* If we're sampling depth and we might need to do shader rewrites for
1109           * legacy shadow sampling, then set up an extra image view that just
1110           * returns the red (depth) component, so you can always have the shadow
1111           * result available in the red component for the in-shader swizzling.
1112           * (Or if we have PVR's needs_zs_shader_swizzle and are sampling ONE
1113           * value for stencil, which also uses that view).
1114           */
1115          if (ivci.subresourceRange.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT ||
1116              zink_screen(ctx->base.screen)->driver_workarounds.needs_zs_shader_swizzle) {
1117             VkComponentSwizzle *swizzle = (VkComponentSwizzle*)&ivci.components;
1118             for (unsigned i = 0; i < 4; i++) {
1119                if (swizzle[i] == VK_COMPONENT_SWIZZLE_ONE ||
1120                    (swizzle[i] == VK_COMPONENT_SWIZZLE_ZERO && ivci.subresourceRange.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT))
1121                   red_depth_sampler_view = true;
1122             }
1123             /* this is the data that will be used in shader rewrites */
1124             sampler_view->swizzle.s[0] = clamp_zs_swizzle(sampler_view->base.swizzle_r);
1125             sampler_view->swizzle.s[1] = clamp_zs_swizzle(sampler_view->base.swizzle_g);
1126             sampler_view->swizzle.s[2] = clamp_zs_swizzle(sampler_view->base.swizzle_b);
1127             sampler_view->swizzle.s[3] = clamp_zs_swizzle(sampler_view->base.swizzle_a);
1128          }
1129       } else {
1130          enum pipe_swizzle swizzle[4] = {
1131             sampler_view->base.swizzle_r,
1132             sampler_view->base.swizzle_g,
1133             sampler_view->base.swizzle_b,
1134             sampler_view->base.swizzle_a
1135          };
1136          /* if we have e.g., R8G8B8X8, then we have to ignore alpha since we're just emulating
1137           * these formats
1138           */
1139          if (zink_format_is_voidable_rgba_variant(state->format)) {
1140             const struct util_format_description *view_desc = util_format_description(state->format);
1141             for (int i = 0; i < 4; ++i)
1142                swizzle[i] = zink_clamp_void_swizzle(view_desc, swizzle[i]);
1143          } else if (util_format_is_alpha(state->format) && res->format != VK_FORMAT_A8_UNORM_KHR) {
1144             for (int i = 0; i < 4; ++i)
1145                swizzle[i] = clamp_alpha_swizzle(swizzle[i]);
1146          } else if (util_format_is_luminance(pres->format) ||
1147                     util_format_is_luminance_alpha(pres->format)) {
1148             if (util_format_is_luminance(pres->format)) {
1149                for (int i = 0; i < 4; ++i)
1150                   swizzle[i] = clamp_luminance_swizzle(swizzle[i]);
1151             } else {
1152                for (int i = 0; i < 4; ++i)
1153                   swizzle[i] = clamp_luminance_alpha_swizzle(swizzle[i]);
1154             }
1155             if (state->format != pres->format) {
1156                /* luminance / luminance-alpha formats can be reinterpreted
1157                 * as red / red-alpha formats by the state-tracker, and we
1158                 * need to whack the green/blue channels here to the
1159                 * correct values for that to work.
1160                 */
1161                enum pipe_format linear = util_format_linear(pres->format);
1162                if (state->format == util_format_luminance_to_red(linear)) {
1163                   assert(swizzle[1] == PIPE_SWIZZLE_X ||
1164                          swizzle[1] == PIPE_SWIZZLE_0);
1165                   assert(swizzle[2] == PIPE_SWIZZLE_X ||
1166                          swizzle[2] == PIPE_SWIZZLE_0);
1167                   swizzle[1] = swizzle[2] = PIPE_SWIZZLE_0;
1168                } else
1169                   assert(state->format == linear);
1170             }
1171          } else if (util_format_is_red_alpha(pres->format)) {
1172             /* RA formats are mapped to RG with adjusted swizzle */
1173             assert(util_format_is_red_green(vk_format_to_pipe_format(ivci.format)));
1174             swizzle[3] = PIPE_SWIZZLE_Y;
1175          }
1176
1177          ivci.components.r = zink_component_mapping(swizzle[0]);
1178          ivci.components.g = zink_component_mapping(swizzle[1]);
1179          ivci.components.b = zink_component_mapping(swizzle[2]);
1180          ivci.components.a = zink_component_mapping(swizzle[3]);
1181       }
1182       assert(ivci.format);
1183
1184       sampler_view->image_view = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci);
1185       if (!screen->info.have_EXT_non_seamless_cube_map && viewtype_is_cube(&sampler_view->image_view->ivci)) {
1186          ivci.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
1187          sampler_view->cube_array = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci);
1188       } else if (red_depth_sampler_view) {
1189          /* there is only one component, and real swizzling can't be done here,
1190           * so ensure the shader gets the sampled data
1191           */
1192          ivci.components.r = VK_COMPONENT_SWIZZLE_R;
1193          ivci.components.g = VK_COMPONENT_SWIZZLE_R;
1194          ivci.components.b = VK_COMPONENT_SWIZZLE_R;
1195          ivci.components.a = VK_COMPONENT_SWIZZLE_R;
1196          sampler_view->zs_view = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci);
1197       }
1198       err = !sampler_view->image_view;
1199    } else {
1200       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
1201          return &sampler_view->base;
1202       VkBufferViewCreateInfo bvci = create_bvci(ctx, res, state->format, state->u.buf.offset, state->u.buf.size);
1203       sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
1204       err = !sampler_view->buffer_view;
1205    }
1206    if (err) {
1207       FREE_CL(sampler_view);
1208       return NULL;
1209    }
1210    return &sampler_view->base;
1211 }
1212
1213 void
1214 zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view)
1215 {
1216    struct zink_resource *res = zink_resource(buffer_view->pres);
1217    simple_mtx_lock(&res->bufferview_mtx);
1218    if (buffer_view->reference.count) {
1219       /* got a cache hit during deletion */
1220       simple_mtx_unlock(&res->bufferview_mtx);
1221       return;
1222    }
1223    struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, buffer_view->hash, &buffer_view->bvci);
1224    assert(he);
1225    _mesa_hash_table_remove(&res->bufferview_cache, he);
1226    simple_mtx_unlock(&res->bufferview_mtx);
1227    simple_mtx_lock(&res->obj->view_lock);
1228    util_dynarray_append(&res->obj->views, VkBufferView, buffer_view->buffer_view);
1229    simple_mtx_unlock(&res->obj->view_lock);
1230    pipe_resource_reference(&buffer_view->pres, NULL);
1231    FREE(buffer_view);
1232 }
1233
1234 static void
1235 zink_sampler_view_destroy(struct pipe_context *pctx,
1236                           struct pipe_sampler_view *pview)
1237 {
1238    struct zink_sampler_view *view = zink_sampler_view(pview);
1239    if (pview->texture->target == PIPE_BUFFER)
1240       zink_buffer_view_reference(zink_screen(pctx->screen), &view->buffer_view, NULL);
1241    else {
1242       zink_surface_reference(zink_screen(pctx->screen), &view->image_view, NULL);
1243       zink_surface_reference(zink_screen(pctx->screen), &view->cube_array, NULL);
1244       zink_surface_reference(zink_screen(pctx->screen), &view->zs_view, NULL);
1245    }
1246    pipe_resource_reference(&pview->texture, NULL);
1247    FREE_CL(view);
1248 }
1249
1250 static void
1251 zink_get_sample_position(struct pipe_context *ctx,
1252                          unsigned sample_count,
1253                          unsigned sample_index,
1254                          float *out_value)
1255 {
1256    /* TODO: handle this I guess */
1257    assert(zink_screen(ctx->screen)->info.props.limits.standardSampleLocations);
1258    u_default_get_sample_position(ctx, sample_count, sample_index, out_value);
1259 }
1260
1261 static void
1262 zink_set_polygon_stipple(struct pipe_context *pctx,
1263                          const struct pipe_poly_stipple *ps)
1264 {
1265 }
1266
1267 ALWAYS_INLINE static void
1268 update_res_bind_count(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool decrement)
1269 {
1270    if (decrement) {
1271       assert(res->bind_count[is_compute]);
1272       if (!--res->bind_count[is_compute])
1273          _mesa_set_remove_key(ctx->need_barriers[is_compute], res);
1274       check_resource_for_batch_ref(ctx, res);
1275    } else
1276       res->bind_count[is_compute]++;
1277 }
1278
1279 ALWAYS_INLINE static void
1280 update_existing_vbo(struct zink_context *ctx, unsigned slot)
1281 {
1282    if (!ctx->vertex_buffers[slot].buffer.resource)
1283       return;
1284    struct zink_resource *res = zink_resource(ctx->vertex_buffers[slot].buffer.resource);
1285    res->vbo_bind_count--;
1286    res->vbo_bind_mask &= ~BITFIELD_BIT(slot);
1287    if (!res->vbo_bind_count) {
1288       res->gfx_barrier &= ~VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1289       res->barrier_access[0] &= ~VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1290    }
1291    update_res_bind_count(ctx, res, false, true);
1292 }
1293
1294 static void
1295 zink_set_vertex_buffers(struct pipe_context *pctx,
1296                         unsigned num_buffers,
1297                         unsigned unbind_num_trailing_slots,
1298                         bool take_ownership,
1299                         const struct pipe_vertex_buffer *buffers)
1300 {
1301    struct zink_context *ctx = zink_context(pctx);
1302    const bool have_input_state = zink_screen(pctx->screen)->info.have_EXT_vertex_input_dynamic_state;
1303    const bool need_state_change = !zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state &&
1304                                   !have_input_state;
1305    uint32_t enabled_buffers = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
1306    enabled_buffers |= u_bit_consecutive(0, num_buffers);
1307    enabled_buffers &= ~u_bit_consecutive(num_buffers, unbind_num_trailing_slots);
1308
1309    if (buffers) {
1310       for (unsigned i = 0; i < num_buffers; ++i) {
1311          const struct pipe_vertex_buffer *vb = buffers + i;
1312          struct pipe_vertex_buffer *ctx_vb = &ctx->vertex_buffers[i];
1313          update_existing_vbo(ctx, i);
1314          if (!take_ownership)
1315             pipe_resource_reference(&ctx_vb->buffer.resource, vb->buffer.resource);
1316          else {
1317             pipe_resource_reference(&ctx_vb->buffer.resource, NULL);
1318             ctx_vb->buffer.resource = vb->buffer.resource;
1319          }
1320          if (vb->buffer.resource) {
1321             struct zink_resource *res = zink_resource(vb->buffer.resource);
1322             res->vbo_bind_mask |= BITFIELD_BIT(i);
1323             res->vbo_bind_count++;
1324             res->gfx_barrier |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
1325             res->barrier_access[0] |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
1326             update_res_bind_count(ctx, res, false, false);
1327             ctx_vb->buffer_offset = vb->buffer_offset;
1328             /* always barrier before possible rebind */
1329             zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
1330                                          VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
1331             zink_batch_resource_usage_set(&ctx->batch, res, false, true);
1332             res->obj->unordered_read = false;
1333          } else {
1334             enabled_buffers &= ~BITFIELD_BIT(i);
1335          }
1336       }
1337    } else {
1338       for (unsigned i = 0; i < num_buffers; ++i) {
1339          update_existing_vbo(ctx, i);
1340          pipe_resource_reference(&ctx->vertex_buffers[i].buffer.resource, NULL);
1341       }
1342    }
1343    for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
1344       update_existing_vbo(ctx, i);
1345       pipe_resource_reference(&ctx->vertex_buffers[i].buffer.resource, NULL);
1346    }
1347    if (need_state_change)
1348       ctx->vertex_state_changed = true;
1349    else if (!have_input_state && ctx->gfx_pipeline_state.vertex_buffers_enabled_mask != enabled_buffers)
1350       ctx->vertex_state_changed = true;
1351    ctx->gfx_pipeline_state.vertex_buffers_enabled_mask = enabled_buffers;
1352    ctx->vertex_buffers_dirty = num_buffers > 0;
1353 #ifndef NDEBUG
1354    u_foreach_bit(b, enabled_buffers)
1355       assert(ctx->vertex_buffers[b].buffer.resource);
1356 #endif
1357 }
1358
1359 static void
1360 zink_set_viewport_states(struct pipe_context *pctx,
1361                          unsigned start_slot,
1362                          unsigned num_viewports,
1363                          const struct pipe_viewport_state *state)
1364 {
1365    struct zink_context *ctx = zink_context(pctx);
1366
1367    for (unsigned i = 0; i < num_viewports; ++i)
1368       ctx->vp_state.viewport_states[start_slot + i] = state[i];
1369
1370    ctx->vp_state_changed = true;
1371    zink_flush_dgc_if_enabled(ctx);
1372 }
1373
1374 static void
1375 zink_set_scissor_states(struct pipe_context *pctx,
1376                         unsigned start_slot, unsigned num_scissors,
1377                         const struct pipe_scissor_state *states)
1378 {
1379    struct zink_context *ctx = zink_context(pctx);
1380
1381    for (unsigned i = 0; i < num_scissors; i++)
1382       ctx->vp_state.scissor_states[start_slot + i] = states[i];
1383    ctx->scissor_changed = true;
1384    zink_flush_dgc_if_enabled(ctx);
1385 }
1386
1387 static void
1388 zink_set_inlinable_constants(struct pipe_context *pctx,
1389                              gl_shader_stage shader,
1390                              uint num_values, uint32_t *values)
1391 {
1392    struct zink_context *ctx = (struct zink_context *)pctx;
1393    const uint32_t bit = BITFIELD_BIT(shader);
1394    uint32_t *inlinable_uniforms;
1395    struct zink_shader_key *key = NULL;
1396
1397    if (shader == MESA_SHADER_COMPUTE) {
1398       key = &ctx->compute_pipeline_state.key;
1399    } else {
1400       assert(!zink_screen(pctx->screen)->optimal_keys ||
1401              (shader == MESA_SHADER_GEOMETRY &&
1402               ctx->gfx_stages[MESA_SHADER_GEOMETRY] &&
1403               ctx->gfx_stages[MESA_SHADER_GEOMETRY]->non_fs.is_generated));
1404       key = &ctx->gfx_pipeline_state.shader_keys.key[shader];
1405    }
1406    inlinable_uniforms = key->base.inlined_uniform_values;
1407    if (!(ctx->inlinable_uniforms_valid_mask & bit) ||
1408        memcmp(inlinable_uniforms, values, num_values * 4)) {
1409       memcpy(inlinable_uniforms, values, num_values * 4);
1410       if (shader == MESA_SHADER_COMPUTE)
1411          ctx->compute_dirty = true;
1412       else
1413          ctx->dirty_gfx_stages |= bit;
1414       ctx->inlinable_uniforms_valid_mask |= bit;
1415       key->inline_uniforms = true;
1416    }
1417 }
1418
1419 ALWAYS_INLINE static void
1420 unbind_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1421 {
1422    if (!res->sampler_binds[pstage] && !res->image_binds[pstage] && !res->all_bindless)
1423       res->gfx_barrier &= ~zink_pipeline_flags_from_pipe_stage(pstage);
1424 }
1425
1426 ALWAYS_INLINE static void
1427 unbind_buffer_descriptor_stage(struct zink_resource *res, gl_shader_stage pstage)
1428 {
1429    if (!res->ubo_bind_mask[pstage] && !res->ssbo_bind_mask[pstage])
1430       unbind_descriptor_stage(res, pstage);
1431 }
1432
1433 ALWAYS_INLINE static void
1434 unbind_ubo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot)
1435 {
1436    if (!res)
1437       return;
1438    res->ubo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1439    res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1440    unbind_buffer_descriptor_stage(res, pstage);
1441    if (!res->ubo_bind_count[pstage == MESA_SHADER_COMPUTE])
1442       res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_UNIFORM_READ_BIT;
1443    update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1444 }
1445
1446 static void
1447 invalidate_inlined_uniforms(struct zink_context *ctx, gl_shader_stage pstage)
1448 {
1449    unsigned bit = BITFIELD_BIT(pstage);
1450    if (!(ctx->inlinable_uniforms_valid_mask & bit))
1451       return;
1452    ctx->inlinable_uniforms_valid_mask &= ~bit;
1453    if (pstage == MESA_SHADER_COMPUTE) {
1454       ctx->compute_dirty = true;
1455       return;
1456    }
1457    assert(!zink_screen(ctx->base.screen)->optimal_keys || (pstage == MESA_SHADER_GEOMETRY && ctx->is_generated_gs_bound));
1458    ctx->dirty_gfx_stages |= bit;
1459    struct zink_shader_key *key = &ctx->gfx_pipeline_state.shader_keys.key[pstage];
1460    key->inline_uniforms = false;
1461 }
1462
1463 static void
1464 zink_set_constant_buffer(struct pipe_context *pctx,
1465                          gl_shader_stage shader, uint index,
1466                          bool take_ownership,
1467                          const struct pipe_constant_buffer *cb)
1468 {
1469    struct zink_context *ctx = zink_context(pctx);
1470    bool update = false;
1471
1472    struct zink_resource *res = zink_resource(ctx->ubos[shader][index].buffer);
1473    if (cb) {
1474       struct pipe_resource *buffer = cb->buffer;
1475       unsigned offset = cb->buffer_offset;
1476       struct zink_screen *screen = zink_screen(pctx->screen);
1477       if (cb->user_buffer) {
1478          u_upload_data(ctx->base.const_uploader, 0, cb->buffer_size,
1479                        screen->info.props.limits.minUniformBufferOffsetAlignment,
1480                        cb->user_buffer, &offset, &buffer);
1481       }
1482       struct zink_resource *new_res = zink_resource(buffer);
1483       if (new_res) {
1484          if (new_res != res) {
1485             unbind_ubo(ctx, res, shader, index);
1486             new_res->ubo_bind_count[shader == MESA_SHADER_COMPUTE]++;
1487             new_res->ubo_bind_mask[shader] |= BITFIELD_BIT(index);
1488             new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader);
1489             new_res->barrier_access[shader == MESA_SHADER_COMPUTE] |= VK_ACCESS_UNIFORM_READ_BIT;
1490             update_res_bind_count(ctx, new_res, shader == MESA_SHADER_COMPUTE, false);
1491          }
1492          zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, VK_ACCESS_UNIFORM_READ_BIT,
1493                                       new_res->gfx_barrier);
1494          zink_batch_resource_usage_set(&ctx->batch, new_res, false, true);
1495          if (!ctx->unordered_blitting)
1496             new_res->obj->unordered_read = false;
1497       }
1498       update |= ctx->ubos[shader][index].buffer_offset != offset ||
1499                 !!res != !!buffer || (res && res->obj->buffer != new_res->obj->buffer) ||
1500                 ctx->ubos[shader][index].buffer_size != cb->buffer_size;
1501
1502       if (take_ownership) {
1503          pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1504          ctx->ubos[shader][index].buffer = buffer;
1505       } else {
1506          pipe_resource_reference(&ctx->ubos[shader][index].buffer, buffer);
1507       }
1508       ctx->ubos[shader][index].buffer_offset = offset;
1509       ctx->ubos[shader][index].buffer_size = cb->buffer_size;
1510       ctx->ubos[shader][index].user_buffer = NULL;
1511
1512       if (cb->user_buffer)
1513          pipe_resource_reference(&buffer, NULL);
1514
1515       if (index + 1 >= ctx->di.num_ubos[shader])
1516          ctx->di.num_ubos[shader] = index + 1;
1517       update_descriptor_state_ubo(ctx, shader, index, new_res);
1518    } else {
1519       ctx->ubos[shader][index].buffer_offset = 0;
1520       ctx->ubos[shader][index].buffer_size = 0;
1521       ctx->ubos[shader][index].user_buffer = NULL;
1522       if (res) {
1523          unbind_ubo(ctx, res, shader, index);
1524          update_descriptor_state_ubo(ctx, shader, index, NULL);
1525       }
1526       update = !!ctx->ubos[shader][index].buffer;
1527
1528       pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1529       if (ctx->di.num_ubos[shader] == index + 1)
1530          ctx->di.num_ubos[shader]--;
1531    }
1532    if (index == 0) {
1533       /* Invalidate current inlinable uniforms. */
1534       invalidate_inlined_uniforms(ctx, shader);
1535    }
1536
1537    if (update)
1538       ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, index, 1);
1539 }
1540
1541 ALWAYS_INLINE static void
1542 unbind_descriptor_reads(struct zink_resource *res, bool is_compute)
1543 {
1544    if (!res->sampler_bind_count[is_compute] && !res->image_bind_count[is_compute] && !res->all_bindless)
1545       res->barrier_access[is_compute] &= ~VK_ACCESS_SHADER_READ_BIT;
1546 }
1547
1548 ALWAYS_INLINE static void
1549 unbind_buffer_descriptor_reads(struct zink_resource *res, bool is_compute)
1550 {
1551    if (!res->ssbo_bind_count[is_compute] && !res->all_bindless)
1552       unbind_descriptor_reads(res, is_compute);
1553 }
1554
1555 ALWAYS_INLINE static void
1556 unbind_ssbo(struct zink_context *ctx, struct zink_resource *res, gl_shader_stage pstage, unsigned slot, bool writable)
1557 {
1558    if (!res)
1559       return;
1560    res->ssbo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1561    res->ssbo_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1562    unbind_buffer_descriptor_stage(res, pstage);
1563    unbind_buffer_descriptor_reads(res, pstage == MESA_SHADER_COMPUTE);
1564    update_res_bind_count(ctx, res, pstage == MESA_SHADER_COMPUTE, true);
1565    if (writable)
1566       res->write_bind_count[pstage == MESA_SHADER_COMPUTE]--;
1567    if (!res->write_bind_count[pstage == MESA_SHADER_COMPUTE])
1568       res->barrier_access[pstage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1569 }
1570
1571 static void
1572 zink_set_shader_buffers(struct pipe_context *pctx,
1573                         gl_shader_stage p_stage,
1574                         unsigned start_slot, unsigned count,
1575                         const struct pipe_shader_buffer *buffers,
1576                         unsigned writable_bitmask)
1577 {
1578    struct zink_context *ctx = zink_context(pctx);
1579    bool update = false;
1580    unsigned max_slot = 0;
1581
1582    unsigned modified_bits = u_bit_consecutive(start_slot, count);
1583    unsigned old_writable_mask = ctx->writable_ssbos[p_stage];
1584    assert(!ctx->unordered_blitting);
1585    ctx->writable_ssbos[p_stage] &= ~modified_bits;
1586    ctx->writable_ssbos[p_stage] |= writable_bitmask << start_slot;
1587
1588    for (unsigned i = 0; i < count; i++) {
1589       unsigned slot = start_slot + i;
1590       struct pipe_shader_buffer *ssbo = &ctx->ssbos[p_stage][slot];
1591       struct zink_resource *res = ssbo->buffer ? zink_resource(ssbo->buffer) : NULL;
1592       bool was_writable = old_writable_mask & BITFIELD64_BIT(slot);
1593       if (buffers && buffers[i].buffer) {
1594          struct zink_resource *new_res = zink_resource(buffers[i].buffer);
1595          if (new_res != res) {
1596             unbind_ssbo(ctx, res, p_stage, slot, was_writable);
1597             new_res->ssbo_bind_mask[p_stage] |= BITFIELD_BIT(slot);
1598             new_res->ssbo_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1599             new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage);
1600             update_res_bind_count(ctx, new_res, p_stage == MESA_SHADER_COMPUTE, false);
1601          }
1602          VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT;
1603          if (ctx->writable_ssbos[p_stage] & BITFIELD64_BIT(slot)) {
1604             new_res->write_bind_count[p_stage == MESA_SHADER_COMPUTE]++;
1605             access |= VK_ACCESS_SHADER_WRITE_BIT;
1606          }
1607          pipe_resource_reference(&ssbo->buffer, &new_res->base.b);
1608          new_res->barrier_access[p_stage == MESA_SHADER_COMPUTE] |= access;
1609          ssbo->buffer_offset = buffers[i].buffer_offset;
1610          ssbo->buffer_size = MIN2(buffers[i].buffer_size, new_res->base.b.width0 - ssbo->buffer_offset);
1611          util_range_add(&new_res->base.b, &new_res->valid_buffer_range, ssbo->buffer_offset,
1612                         ssbo->buffer_offset + ssbo->buffer_size);
1613          zink_screen(ctx->base.screen)->buffer_barrier(ctx, new_res, access,
1614                                       new_res->gfx_barrier);
1615          zink_batch_resource_usage_set(&ctx->batch, new_res, access & VK_ACCESS_SHADER_WRITE_BIT, true);
1616          update = true;
1617          max_slot = MAX2(max_slot, slot);
1618          update_descriptor_state_ssbo(ctx, p_stage, slot, new_res);
1619          if (zink_resource_access_is_write(access))
1620             new_res->obj->unordered_write = false;
1621          new_res->obj->unordered_read = false;
1622       } else {
1623          if (res)
1624             update = true;
1625          ssbo->buffer_offset = 0;
1626          ssbo->buffer_size = 0;
1627          if (res) {
1628             unbind_ssbo(ctx, res, p_stage, slot, was_writable);
1629             update_descriptor_state_ssbo(ctx, p_stage, slot, NULL);
1630          }
1631          pipe_resource_reference(&ssbo->buffer, NULL);
1632       }
1633    }
1634    if (start_slot + count >= ctx->di.num_ssbos[p_stage])
1635       ctx->di.num_ssbos[p_stage] = max_slot + 1;
1636    if (update)
1637       ctx->invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_SSBO, start_slot, count);
1638 }
1639
1640 static void
1641 update_binds_for_samplerviews(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1642 {
1643     VkImageLayout layout = get_layout_for_binding(ctx, res, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, is_compute);
1644     if (is_compute) {
1645        u_foreach_bit(slot, res->sampler_binds[MESA_SHADER_COMPUTE]) {
1646           if (ctx->di.textures[MESA_SHADER_COMPUTE][slot].imageLayout != layout) {
1647              update_descriptor_state_sampler(ctx, MESA_SHADER_COMPUTE, slot, res);
1648              ctx->invalidate_descriptor_state(ctx, MESA_SHADER_COMPUTE, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1649           }
1650        }
1651     } else {
1652        for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++) {
1653           u_foreach_bit(slot, res->sampler_binds[i]) {
1654              if (ctx->di.textures[i][slot].imageLayout != layout) {
1655                 update_descriptor_state_sampler(ctx, i, slot, res);
1656                 ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1657              }
1658           }
1659        }
1660     }
1661 }
1662
1663 static void
1664 flush_pending_clears(struct zink_context *ctx, struct zink_resource *res)
1665 {
1666    if (res->fb_bind_count && ctx->clears_enabled)
1667       zink_fb_clears_apply(ctx, &res->base.b);
1668 }
1669
1670 static inline void
1671 unbind_shader_image_counts(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool writable)
1672 {
1673    update_res_bind_count(ctx, res, is_compute, true);
1674    if (writable)
1675       res->write_bind_count[is_compute]--;
1676    res->image_bind_count[is_compute]--;
1677    /* if this was the last image bind, the sampler bind layouts must be updated */
1678    if (!res->obj->is_buffer && !res->image_bind_count[is_compute] && res->bind_count[is_compute])
1679       update_binds_for_samplerviews(ctx, res, is_compute);
1680 }
1681
1682 ALWAYS_INLINE static bool
1683 check_for_layout_update(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1684 {
1685    VkImageLayout layout = res->bind_count[is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1686    VkImageLayout other_layout = res->bind_count[!is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, !is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1687    bool ret = false;
1688    if (!is_compute && res->fb_binds && !(ctx->feedback_loops & res->fb_binds)) {
1689       /* always double check feedback loops */
1690       ret = !!_mesa_set_add(ctx->need_barriers[0], res);
1691    } else {
1692       if (res->bind_count[is_compute] && layout && res->layout != layout)
1693          ret = !!_mesa_set_add(ctx->need_barriers[is_compute], res);
1694       if (res->bind_count[!is_compute] && other_layout && (layout != other_layout || res->layout != other_layout))
1695          ret = !!_mesa_set_add(ctx->need_barriers[!is_compute], res);
1696    }
1697    return ret;
1698 }
1699
1700 static void
1701 unbind_shader_image(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
1702 {
1703    struct zink_image_view *image_view = &ctx->image_views[stage][slot];
1704    bool is_compute = stage == MESA_SHADER_COMPUTE;
1705    if (!image_view->base.resource)
1706       return;
1707
1708    struct zink_resource *res = zink_resource(image_view->base.resource);
1709    res->image_binds[stage] &= ~BITFIELD_BIT(slot);
1710    unbind_shader_image_counts(ctx, res, is_compute, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE);
1711    if (!res->write_bind_count[is_compute])
1712       res->barrier_access[stage == MESA_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1713
1714    if (image_view->base.resource->target == PIPE_BUFFER) {
1715       unbind_buffer_descriptor_stage(res, stage);
1716       unbind_buffer_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1717       zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
1718       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
1719          pipe_resource_reference(&image_view->base.resource, NULL);
1720    } else {
1721       unbind_descriptor_stage(res, stage);
1722       unbind_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1723       if (!res->image_bind_count[is_compute])
1724          check_for_layout_update(ctx, res, is_compute);
1725       zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
1726    }
1727    image_view->base.resource = NULL;
1728    image_view->surface = NULL;
1729 }
1730
1731 static struct zink_buffer_view *
1732 create_image_bufferview(struct zink_context *ctx, const struct pipe_image_view *view)
1733 {
1734    struct zink_resource *res = zink_resource(view->resource);
1735    VkBufferViewCreateInfo bvci = create_bvci(ctx, res, view->format, view->u.buf.offset, view->u.buf.size);
1736    struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1737    if (!buffer_view)
1738       return NULL;
1739    util_range_add(&res->base.b, &res->valid_buffer_range, view->u.buf.offset,
1740                   view->u.buf.offset + view->u.buf.size);
1741    return buffer_view;
1742 }
1743
1744 static void
1745 finalize_image_bind(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1746 {
1747    /* if this is the first image bind and there are sampler binds, the image's sampler layout
1748     * must be updated to GENERAL
1749     */
1750    if (res->image_bind_count[is_compute] == 1 &&
1751        res->bind_count[is_compute] > 1)
1752       update_binds_for_samplerviews(ctx, res, is_compute);
1753    if (!check_for_layout_update(ctx, res, is_compute)) {
1754       /* no deferred barrier: unset unordered usage immediately */
1755       // TODO: figure out a way to link up layouts between unordered and main cmdbuf
1756       // if (zink_resource_access_is_write(res->barrier_access[is_compute]))
1757       res->obj->unordered_write = false;
1758       res->obj->unordered_read = false;
1759    }
1760 }
1761
1762 static struct zink_surface *
1763 create_image_surface(struct zink_context *ctx, const struct pipe_image_view *view, bool is_compute)
1764 {
1765    struct zink_screen *screen = zink_screen(ctx->base.screen);
1766    struct zink_resource *res = zink_resource(view->resource);
1767    struct pipe_surface tmpl = {0};
1768    enum pipe_texture_target target = res->base.b.target;
1769    tmpl.format = view->format;
1770    tmpl.u.tex.level = view->u.tex.level;
1771    tmpl.u.tex.first_layer = view->u.tex.first_layer;
1772    tmpl.u.tex.last_layer = view->u.tex.last_layer;
1773    unsigned depth = 1 + tmpl.u.tex.last_layer - tmpl.u.tex.first_layer;
1774    switch (target) {
1775    case PIPE_TEXTURE_3D:
1776       if (depth < u_minify(res->base.b.depth0, view->u.tex.level)) {
1777          assert(depth == 1);
1778          target = PIPE_TEXTURE_2D;
1779          if (!screen->info.have_EXT_image_2d_view_of_3d ||
1780              !screen->info.view2d_feats.image2DViewOf3D) {
1781             static bool warned = false;
1782             warn_missing_feature(warned, "image2DViewOf3D");
1783          }
1784       } else {
1785          assert(tmpl.u.tex.first_layer == 0);
1786          tmpl.u.tex.last_layer = 0;
1787       }
1788       break;
1789    case PIPE_TEXTURE_2D_ARRAY:
1790    case PIPE_TEXTURE_1D_ARRAY:
1791       if (depth < res->base.b.array_size && depth == 1)
1792          target = target == PIPE_TEXTURE_2D_ARRAY ? PIPE_TEXTURE_2D : PIPE_TEXTURE_1D;
1793       break;
1794    default: break;
1795    }
1796    if (zink_format_needs_mutable(view->resource->format, view->format))
1797       /* mutable not set by default */
1798       zink_resource_object_init_mutable(ctx, res);
1799    VkImageViewCreateInfo ivci = create_ivci(screen, res, &tmpl, target);
1800    struct pipe_surface *psurf = zink_get_surface(ctx, view->resource, &tmpl, &ivci);
1801    if (!psurf)
1802       return NULL;
1803    struct zink_surface *surface = zink_surface(psurf);
1804    if (is_compute)
1805       flush_pending_clears(ctx, res);
1806    return surface;
1807 }
1808
1809 static void
1810 zink_set_shader_images(struct pipe_context *pctx,
1811                        gl_shader_stage shader_type,
1812                        unsigned start_slot, unsigned count,
1813                        unsigned unbind_num_trailing_slots,
1814                        const struct pipe_image_view *images)
1815 {
1816    struct zink_context *ctx = zink_context(pctx);
1817    struct zink_screen *screen = zink_screen(pctx->screen);
1818    bool update = false;
1819    bool is_compute = shader_type == MESA_SHADER_COMPUTE;
1820    assert(!ctx->unordered_blitting);
1821    for (unsigned i = 0; i < count; i++) {
1822       struct zink_image_view *a = &ctx->image_views[shader_type][start_slot + i];
1823       const struct pipe_image_view *b = images ? &images[i] : NULL;
1824       struct zink_resource *res = b ? zink_resource(b->resource) : NULL;
1825       if (b && b->resource) {
1826          if (!zink_resource_object_init_storage(ctx, res)) {
1827             debug_printf("couldn't create storage image!");
1828             continue;
1829          }
1830
1831          VkAccessFlags access = 0;
1832          if (b->access & PIPE_IMAGE_ACCESS_WRITE) {
1833             access |= VK_ACCESS_SHADER_WRITE_BIT;
1834          }
1835          if (b->access & PIPE_IMAGE_ACCESS_READ) {
1836             access |= VK_ACCESS_SHADER_READ_BIT;
1837          }
1838
1839          bool changed = false;
1840          if (!a->base.resource || a->base.resource != b->resource) {
1841             /* this needs a full unbind+bind */
1842             changed = true;
1843             unbind_shader_image(ctx, shader_type, start_slot + i);
1844             update_res_bind_count(ctx, res, is_compute, false);
1845             res->image_bind_count[is_compute]++;
1846             /* always increment write_bind_count on new bind */
1847             if (b->access & PIPE_IMAGE_ACCESS_WRITE)
1848                res->write_bind_count[is_compute]++;
1849             /* db mode refcounts these */
1850             if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB && b->resource->target == PIPE_BUFFER)
1851                pipe_resource_reference(&a->base.resource, b->resource);
1852          } else {
1853             /* resource matches: check for write flag change and partial rebind */
1854
1855             /* previous bind didn't have write: increment */
1856             if ((b->access & PIPE_IMAGE_ACCESS_WRITE) && !(a->base.access & PIPE_IMAGE_ACCESS_WRITE))
1857                res->write_bind_count[is_compute]++;
1858             /* previous bind had write: decrement */
1859             else if (!(b->access & PIPE_IMAGE_ACCESS_WRITE) && (a->base.access & PIPE_IMAGE_ACCESS_WRITE)) {
1860                res->write_bind_count[is_compute]--;
1861                if (!res->write_bind_count[is_compute])
1862                   res->barrier_access[is_compute] &= ~VK_ACCESS_SHADER_WRITE_BIT;
1863             }
1864
1865             /* this may need a partial rebind */
1866             changed = a->base.format != b->format || zink_resource(a->base.resource)->obj != res->obj;
1867             if (!changed) {
1868                if (b->resource->target == PIPE_BUFFER) {
1869                   /* db mode has no partial rebind */
1870                   if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB)
1871                      changed = !!memcmp(&a->base.u.buf, &b->u.buf, sizeof(b->u.buf));
1872                } else {
1873                   /* no memcmp, these are bitfields */
1874                   changed = a->base.u.tex.first_layer != b->u.tex.first_layer ||
1875                             a->base.u.tex.last_layer != b->u.tex.last_layer ||
1876                             a->base.u.tex.level != b->u.tex.level;
1877                }
1878             }
1879          }
1880
1881          if (changed) {
1882             /* this is a partial rebind */
1883             if (b->resource->target == PIPE_BUFFER) {
1884                /* db has no partial rebind */
1885                if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
1886                   /* bufferview rebind: get updated bufferview and unref old one */
1887                   struct zink_buffer_view *bv = create_image_bufferview(ctx, b);
1888                   /* identical rebind was already checked above */
1889                   assert(bv && bv != a->buffer_view);
1890                   zink_buffer_view_reference(screen, &a->buffer_view, NULL);
1891                   /* ref already added by create */
1892                   a->buffer_view = bv;
1893                }
1894                if (zink_resource_access_is_write(access))
1895                   res->obj->unordered_write = false;
1896                res->obj->unordered_read = false;
1897             } else {
1898                /* image rebind: get updated surface and unref old one */
1899                struct zink_surface *surface = create_image_surface(ctx, b, is_compute);
1900                /* identical rebind was already checked above */
1901                assert(surface && surface != a->surface);
1902                zink_surface_reference(screen, &a->surface, NULL);
1903                /* ref already added by create */
1904                a->surface = surface;
1905             }
1906          }
1907
1908          /* these operations occur regardless of binding/rebinding */
1909          res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type);
1910          res->barrier_access[is_compute] |= access;
1911          if (b->resource->target == PIPE_BUFFER) {
1912             screen->buffer_barrier(ctx, res, access,
1913                                          res->gfx_barrier);
1914             zink_batch_resource_usage_set(&ctx->batch, res,
1915                                           zink_resource_access_is_write(access), true);
1916          } else {
1917             finalize_image_bind(ctx, res, is_compute);
1918             zink_batch_resource_usage_set(&ctx->batch, res,
1919                                           zink_resource_access_is_write(access), false);
1920          }
1921          memcpy(&a->base, images + i, sizeof(struct pipe_image_view));
1922          update = true;
1923          res->image_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
1924       } else if (a->base.resource) {
1925          update = true;
1926          unbind_shader_image(ctx, shader_type, start_slot + i);
1927       }
1928       update_descriptor_state_image(ctx, shader_type, start_slot + i, res);
1929    }
1930    for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
1931       update |= !!ctx->image_views[shader_type][start_slot + count + i].base.resource;
1932       unbind_shader_image(ctx, shader_type, start_slot + count + i);
1933       update_descriptor_state_image(ctx, shader_type, start_slot + count + i, NULL);
1934    }
1935    ctx->di.num_images[shader_type] = start_slot + count;
1936    if (update)
1937       ctx->invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_IMAGE, start_slot, count);
1938 }
1939
1940 static void
1941 update_feedback_loop_dynamic_state(struct zink_context *ctx)
1942 {
1943    if (!zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_dynamic_state)
1944       return;
1945    VkImageAspectFlags aspects = 0;
1946    if (ctx->feedback_loops & BITFIELD_MASK(PIPE_MAX_COLOR_BUFS))
1947       aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
1948    if (ctx->feedback_loops & BITFIELD_BIT(PIPE_MAX_COLOR_BUFS))
1949       aspects |= VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
1950    VKCTX(CmdSetAttachmentFeedbackLoopEnableEXT)(ctx->batch.state->cmdbuf, aspects);
1951 }
1952
1953 static void
1954 update_feedback_loop_state(struct zink_context *ctx, unsigned idx, unsigned feedback_loops)
1955 {
1956    if (feedback_loops != ctx->feedback_loops) {
1957       if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
1958          if (ctx->gfx_pipeline_state.feedback_loop_zs)
1959             ctx->gfx_pipeline_state.dirty = true;
1960          ctx->gfx_pipeline_state.feedback_loop_zs = false;
1961       } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
1962          if (ctx->gfx_pipeline_state.feedback_loop)
1963             ctx->gfx_pipeline_state.dirty = true;
1964          ctx->gfx_pipeline_state.feedback_loop = false;
1965       }
1966       update_feedback_loop_dynamic_state(ctx);
1967    }
1968    ctx->feedback_loops = feedback_loops;
1969 }
1970
1971 ALWAYS_INLINE static void
1972 unbind_samplerview(struct zink_context *ctx, gl_shader_stage stage, unsigned slot)
1973 {
1974    struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][slot]);
1975    if (!sv || !sv->base.texture)
1976       return;
1977    struct zink_resource *res = zink_resource(sv->base.texture);
1978    res->sampler_bind_count[stage == MESA_SHADER_COMPUTE]--;
1979    if (stage != MESA_SHADER_COMPUTE && !res->sampler_bind_count[0] && res->fb_bind_count) {
1980       u_foreach_bit(idx, res->fb_binds) {
1981          if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
1982             ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
1983             ctx->rp_layout_changed = true;
1984          }
1985          update_feedback_loop_state(ctx, idx, ctx->feedback_loops & ~BITFIELD_BIT(idx));
1986       }
1987    }
1988    update_res_bind_count(ctx, res, stage == MESA_SHADER_COMPUTE, true);
1989    res->sampler_binds[stage] &= ~BITFIELD_BIT(slot);
1990    if (res->obj->is_buffer) {
1991       unbind_buffer_descriptor_stage(res, stage);
1992       unbind_buffer_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1993    } else {
1994       unbind_descriptor_stage(res, stage);
1995       unbind_descriptor_reads(res, stage == MESA_SHADER_COMPUTE);
1996       if (!res->sampler_bind_count[stage == MESA_SHADER_COMPUTE])
1997          check_for_layout_update(ctx, res, stage == MESA_SHADER_COMPUTE);
1998    }
1999    assert(slot < 32);
2000    ctx->di.zs_swizzle[stage].mask &= ~BITFIELD_BIT(slot);
2001 }
2002
2003 static void
2004 zink_set_sampler_views(struct pipe_context *pctx,
2005                        gl_shader_stage shader_type,
2006                        unsigned start_slot,
2007                        unsigned num_views,
2008                        unsigned unbind_num_trailing_slots,
2009                        bool take_ownership,
2010                        struct pipe_sampler_view **views)
2011 {
2012    struct zink_context *ctx = zink_context(pctx);
2013
2014    const uint32_t mask = BITFIELD_RANGE(start_slot, num_views);
2015    uint32_t shadow_mask = ctx->di.zs_swizzle[shader_type].mask;
2016    ctx->di.cubes[shader_type] &= ~mask;
2017
2018    bool update = false;
2019    bool shadow_update = false;
2020    if (views) {
2021       for (unsigned i = 0; i < num_views; ++i) {
2022          struct pipe_sampler_view *pview = views[i];
2023          struct zink_sampler_view *a = zink_sampler_view(ctx->sampler_views[shader_type][start_slot + i]);
2024          struct zink_sampler_view *b = zink_sampler_view(pview);
2025
2026          if (a == b) {
2027             if (take_ownership) {
2028                struct pipe_sampler_view *view = views[i];
2029                pipe_sampler_view_reference(&view, NULL);
2030             }
2031             continue;
2032          }
2033
2034          struct zink_resource *res = b ? zink_resource(b->base.texture) : NULL;
2035          if (b && b->base.texture) {
2036             if (!a || zink_resource(a->base.texture) != res) {
2037                if (a)
2038                   unbind_samplerview(ctx, shader_type, start_slot + i);
2039                update_res_bind_count(ctx, res, shader_type == MESA_SHADER_COMPUTE, false);
2040                res->sampler_bind_count[shader_type == MESA_SHADER_COMPUTE]++;
2041                res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type);
2042                res->barrier_access[shader_type == MESA_SHADER_COMPUTE] |= VK_ACCESS_SHADER_READ_BIT;
2043             }
2044             if (res->base.b.target == PIPE_BUFFER) {
2045                if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2046                   if (!a || a->base.texture != b->base.texture || zink_resource(a->base.texture)->obj != res->obj ||
2047                      memcmp(&a->base.u.buf, &b->base.u.buf, sizeof(b->base.u.buf)))
2048                      update = true;
2049                } else if (b->buffer_view->bvci.buffer != res->obj->buffer) {
2050                   /* if this resource has been rebound while it wasn't set here,
2051                   * its backing resource will have changed and thus we need to update
2052                   * the bufferview
2053                   */
2054                   VkBufferViewCreateInfo bvci = b->buffer_view->bvci;
2055                   bvci.buffer = res->obj->buffer;
2056                   struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
2057                   assert(buffer_view != b->buffer_view);
2058                   zink_buffer_view_reference(zink_screen(ctx->base.screen), &b->buffer_view, NULL);
2059                   b->buffer_view = buffer_view;
2060                   update = true;
2061                } else if (!a || a->buffer_view->buffer_view != b->buffer_view->buffer_view)
2062                      update = true;
2063                zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT,
2064                                           res->gfx_barrier);
2065                zink_batch_resource_usage_set(&ctx->batch, res, false, true);
2066                if (!ctx->unordered_blitting)
2067                   res->obj->unordered_read = false;
2068             } else {
2069                if (zink_format_needs_mutable(res->base.b.format, b->image_view->base.format))
2070                   /* mutable not set by default */
2071                   zink_resource_object_init_mutable(ctx, res);
2072                if (res->obj != b->image_view->obj) {
2073                   struct pipe_surface *psurf = &b->image_view->base;
2074                   VkImageView iv = b->image_view->image_view;
2075                   zink_rebind_surface(ctx, &psurf);
2076                   b->image_view = zink_surface(psurf);
2077                   update |= iv != b->image_view->image_view;
2078                } else  if (a != b)
2079                   update = true;
2080                if (shader_type == MESA_SHADER_COMPUTE)
2081                   flush_pending_clears(ctx, res);
2082                if (b->cube_array) {
2083                   ctx->di.cubes[shader_type] |= BITFIELD_BIT(start_slot + i);
2084                }
2085                if (!check_for_layout_update(ctx, res, shader_type == MESA_SHADER_COMPUTE) && !ctx->unordered_blitting) {
2086                   /* no deferred barrier: unset unordered usage immediately */
2087                   res->obj->unordered_read = false;
2088                   // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2089                   res->obj->unordered_write = false;
2090                }
2091                if (!a)
2092                   update = true;
2093                zink_batch_resource_usage_set(&ctx->batch, res, false, false);
2094                if (b->zs_view) {
2095                   assert(start_slot + i < 32); //bitfield size
2096                   ctx->di.zs_swizzle[shader_type].mask |= BITFIELD_BIT(start_slot + i);
2097                   /* this is already gonna be slow, so don't bother trying to micro-optimize */
2098                   shadow_update |= memcmp(&ctx->di.zs_swizzle[shader_type].swizzle[start_slot + i],
2099                                           &b->swizzle, sizeof(struct zink_zs_swizzle));
2100                   memcpy(&ctx->di.zs_swizzle[shader_type].swizzle[start_slot + i], &b->swizzle, sizeof(struct zink_zs_swizzle));
2101                } else {
2102                   assert(start_slot + i < 32); //bitfield size
2103                   ctx->di.zs_swizzle[shader_type].mask &= ~BITFIELD_BIT(start_slot + i);
2104                }
2105             }
2106             res->sampler_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
2107          } else if (a) {
2108             unbind_samplerview(ctx, shader_type, start_slot + i);
2109             update = true;
2110          }
2111          if (take_ownership) {
2112             pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], NULL);
2113             ctx->sampler_views[shader_type][start_slot + i] = pview;
2114          } else {
2115             pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], pview);
2116          }
2117          update_descriptor_state_sampler(ctx, shader_type, start_slot + i, res);
2118       }
2119    } else {
2120       unbind_num_trailing_slots += num_views;
2121       num_views = 0;
2122    }
2123    for (unsigned i = 0; i < unbind_num_trailing_slots; ++i) {
2124       unsigned slot = start_slot + num_views + i;
2125       update |= !!ctx->sampler_views[shader_type][slot];
2126       unbind_samplerview(ctx, shader_type, slot);
2127       pipe_sampler_view_reference(
2128          &ctx->sampler_views[shader_type][slot],
2129          NULL);
2130       update_descriptor_state_sampler(ctx, shader_type, slot, NULL);
2131    }
2132    ctx->di.num_sampler_views[shader_type] = start_slot + num_views;
2133    if (update) {
2134       struct zink_screen *screen = zink_screen(pctx->screen);
2135       ctx->invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, num_views);
2136       if (!screen->info.have_EXT_non_seamless_cube_map)
2137          update_nonseamless_shader_key(ctx, shader_type);
2138       shadow_update |= shadow_mask != ctx->di.zs_swizzle[shader_type].mask;
2139       zink_set_zs_needs_shader_swizzle_key(ctx, shader_type, shadow_update);
2140    }
2141 }
2142
2143 static uint64_t
2144 zink_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *state)
2145 {
2146    struct zink_context *ctx = zink_context(pctx);
2147    struct zink_resource *res = zink_resource(view->texture);
2148    struct zink_sampler_view *sv = zink_sampler_view(view);
2149    struct zink_bindless_descriptor *bd;
2150    bd = calloc(1, sizeof(struct zink_bindless_descriptor));
2151    if (!bd)
2152       return 0;
2153
2154    bd->sampler = pctx->create_sampler_state(pctx, state);
2155    if (!bd->sampler) {
2156       free(bd);
2157       return 0;
2158    }
2159
2160    bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
2161    if (res->base.b.target == PIPE_BUFFER) {
2162       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2163          pipe_resource_reference(&bd->ds.db.pres, view->texture);
2164          bd->ds.db.format = view->format;
2165          bd->ds.db.offset = view->u.buf.offset;
2166          bd->ds.db.size = view->u.buf.size;
2167       } else {
2168          zink_buffer_view_reference(zink_screen(pctx->screen), &bd->ds.bufferview, sv->buffer_view);
2169       }
2170    } else {
2171       zink_surface_reference(zink_screen(pctx->screen), &bd->ds.surface, sv->image_view);
2172    }
2173    uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].tex_slots);
2174    if (bd->ds.is_buffer)
2175       handle += ZINK_MAX_BINDLESS_HANDLES;
2176    bd->handle = handle;
2177    _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].tex_handles, (void*)(uintptr_t)handle, bd);
2178    return handle;
2179 }
2180
2181 static void
2182 zink_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
2183 {
2184    struct zink_context *ctx = zink_context(pctx);
2185    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2186    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
2187    assert(he);
2188    struct zink_bindless_descriptor *bd = he->data;
2189    struct zink_descriptor_surface *ds = &bd->ds;
2190    _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].tex_handles, he);
2191    uint32_t h = handle;
2192    util_dynarray_append(&ctx->batch.state->bindless_releases[0], uint32_t, h);
2193
2194    if (ds->is_buffer) {
2195       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2196          pipe_resource_reference(&ds->db.pres, NULL);
2197       } else {
2198          zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
2199       }
2200    } else {
2201       zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
2202       pctx->delete_sampler_state(pctx, bd->sampler);
2203    }
2204    free(ds);
2205 }
2206
2207 static void
2208 rebind_bindless_bufferview(struct zink_context *ctx, struct zink_resource *res, struct zink_descriptor_surface *ds)
2209 {
2210    /* descriptor buffer is unaffected by this */
2211    if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
2212       return;
2213    /* if this resource has been rebound while it wasn't set here,
2214     * its backing resource will have changed and thus we need to update
2215     * the bufferview
2216     */
2217    VkBufferViewCreateInfo bvci = ds->bufferview->bvci;
2218    bvci.buffer = res->obj->buffer;
2219    struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
2220    assert(buffer_view != ds->bufferview);
2221    zink_buffer_view_reference(zink_screen(ctx->base.screen), &ds->bufferview, NULL);
2222    ds->bufferview = buffer_view;
2223 }
2224
2225 static void
2226 zero_bindless_descriptor(struct zink_context *ctx, uint32_t handle, bool is_buffer, bool is_image)
2227 {
2228    if (likely(zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor)) {
2229       if (is_buffer) {
2230          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2231             ctx->di.bindless[is_image].db.buffer_infos[handle].address = 0;
2232             ctx->di.bindless[is_image].db.buffer_infos[handle].range = 0;
2233          } else {
2234             VkBufferView *bv = &ctx->di.bindless[is_image].t.buffer_infos[handle];
2235             *bv = VK_NULL_HANDLE;
2236          }
2237       } else {
2238          VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
2239          memset(ii, 0, sizeof(*ii));
2240       }
2241    } else {
2242       if (is_buffer) {
2243          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2244             ctx->di.bindless[is_image].db.buffer_infos[handle].address = zink_resource(ctx->dummy_bufferview->pres)->obj->bda;
2245             ctx->di.bindless[is_image].db.buffer_infos[handle].range = 1;
2246          } else {
2247             VkBufferView *bv = &ctx->di.bindless[is_image].t.buffer_infos[handle];
2248             struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
2249             *bv = null_bufferview->buffer_view;
2250          }
2251       } else {
2252          struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0);
2253          VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
2254          ii->sampler = VK_NULL_HANDLE;
2255          ii->imageView = null_surface->image_view;
2256          ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2257       }
2258    }
2259 }
2260
2261 static void
2262 unbind_bindless_descriptor(struct zink_context *ctx, struct zink_resource *res)
2263 {
2264    if (!res->bindless[1]) {
2265       /* check to remove write access */
2266       for (unsigned i = 0; i < 2; i++) {
2267          if (!res->write_bind_count[i])
2268             res->barrier_access[i] &= ~VK_ACCESS_SHADER_WRITE_BIT;
2269       }
2270    }
2271    bool is_buffer = res->base.b.target == PIPE_BUFFER;
2272    if (!res->all_bindless) {
2273       /* check to remove read access */
2274       if (is_buffer) {
2275          for (unsigned i = 0; i < 2; i++)
2276             unbind_buffer_descriptor_reads(res, i);
2277       } else {
2278          for (unsigned i = 0; i < 2; i++)
2279             unbind_descriptor_reads(res, i);
2280       }
2281    }
2282    for (unsigned i = 0; i < 2; i++) {
2283       if (!res->image_bind_count[i])
2284          check_for_layout_update(ctx, res, i);
2285    }
2286 }
2287
2288 static void
2289 zink_make_texture_handle_resident(struct pipe_context *pctx, uint64_t handle, bool resident)
2290 {
2291    struct zink_context *ctx = zink_context(pctx);
2292    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2293    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
2294    assert(he);
2295    struct zink_bindless_descriptor *bd = he->data;
2296    struct zink_descriptor_surface *ds = &bd->ds;
2297    struct zink_resource *res = zink_descriptor_surface_resource(ds);
2298    if (is_buffer)
2299       handle -= ZINK_MAX_BINDLESS_HANDLES;
2300    if (resident) {
2301       update_res_bind_count(ctx, res, false, false);
2302       update_res_bind_count(ctx, res, true, false);
2303       res->bindless[0]++;
2304       if (is_buffer) {
2305          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2306             ctx->di.bindless[0].db.buffer_infos[handle].address = res->obj->bda + ds->db.offset;
2307             ctx->di.bindless[0].db.buffer_infos[handle].range = ds->db.size;
2308             ctx->di.bindless[0].db.buffer_infos[handle].format = zink_get_format(zink_screen(ctx->base.screen), ds->db.format);
2309          } else {
2310             if (ds->bufferview->bvci.buffer != res->obj->buffer)
2311                rebind_bindless_bufferview(ctx, res, ds);
2312             VkBufferView *bv = &ctx->di.bindless[0].t.buffer_infos[handle];
2313             *bv = ds->bufferview->buffer_view;
2314          }
2315          zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2316          zink_batch_resource_usage_set(&ctx->batch, res, false, true);
2317          res->obj->unordered_read = false;
2318       } else {
2319          VkDescriptorImageInfo *ii = &ctx->di.bindless[0].img_infos[handle];
2320          ii->sampler = bd->sampler->sampler;
2321          ii->imageView = ds->surface->image_view;
2322          ii->imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
2323          flush_pending_clears(ctx, res);
2324          if (!check_for_layout_update(ctx, res, false)) {
2325             res->obj->unordered_read = false;
2326             // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2327             res->obj->unordered_write = false;
2328          }
2329          if (!check_for_layout_update(ctx, res, true)) {
2330             res->obj->unordered_read = false;
2331             // TODO: figure out a way to link up layouts between unordered and main cmdbuf
2332             res->obj->unordered_write = false;
2333          }
2334          zink_batch_resource_usage_set(&ctx->batch, res, false, false);
2335          res->obj->unordered_write = false;
2336       }
2337       res->gfx_barrier |= VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2338       res->barrier_access[0] |= VK_ACCESS_SHADER_READ_BIT;
2339       res->barrier_access[1] |= VK_ACCESS_SHADER_READ_BIT;
2340       util_dynarray_append(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2341       uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2342       util_dynarray_append(&ctx->di.bindless[0].updates, uint32_t, h);
2343    } else {
2344       zero_bindless_descriptor(ctx, handle, is_buffer, false);
2345       util_dynarray_delete_unordered(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
2346       update_res_bind_count(ctx, res, false, true);
2347       update_res_bind_count(ctx, res, true, true);
2348       res->bindless[0]--;
2349       unbind_bindless_descriptor(ctx, res);
2350    }
2351    ctx->di.bindless_dirty[0] = true;
2352 }
2353
2354 static uint64_t
2355 zink_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
2356 {
2357    struct zink_context *ctx = zink_context(pctx);
2358    struct zink_resource *res = zink_resource(view->resource);
2359    struct zink_bindless_descriptor *bd;
2360    if (!zink_resource_object_init_storage(ctx, res)) {
2361       debug_printf("couldn't create storage image!");
2362       return 0;
2363    }
2364    bd = malloc(sizeof(struct zink_bindless_descriptor));
2365    if (!bd)
2366       return 0;
2367    bd->sampler = NULL;
2368
2369    bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
2370    if (res->base.b.target == PIPE_BUFFER)
2371       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2372          pipe_resource_reference(&bd->ds.db.pres, view->resource);
2373          bd->ds.db.format = view->format;
2374          bd->ds.db.offset = view->u.buf.offset;
2375          bd->ds.db.size = view->u.buf.size;
2376       } else {
2377          bd->ds.bufferview = create_image_bufferview(ctx, view);
2378       }
2379    else
2380       bd->ds.surface = create_image_surface(ctx, view, false);
2381    uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].img_slots);
2382    if (bd->ds.is_buffer)
2383       handle += ZINK_MAX_BINDLESS_HANDLES;
2384    bd->handle = handle;
2385    _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].img_handles, (void*)(uintptr_t)handle, bd);
2386    return handle;
2387 }
2388
2389 static void
2390 zink_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
2391 {
2392    struct zink_context *ctx = zink_context(pctx);
2393    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2394    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2395    assert(he);
2396    struct zink_descriptor_surface *ds = he->data;
2397    _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].img_handles, he);
2398    uint32_t h = handle;
2399    util_dynarray_append(&ctx->batch.state->bindless_releases[1], uint32_t, h);
2400
2401    if (ds->is_buffer) {
2402       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2403          pipe_resource_reference(&ds->db.pres, NULL);
2404       } else {
2405          zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
2406       }
2407    } else {
2408       zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
2409    }
2410    free(ds);
2411 }
2412
2413 static void
2414 zink_make_image_handle_resident(struct pipe_context *pctx, uint64_t handle, unsigned paccess, bool resident)
2415 {
2416    struct zink_context *ctx = zink_context(pctx);
2417    bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
2418    struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
2419    assert(he);
2420    struct zink_bindless_descriptor *bd = he->data;
2421    struct zink_descriptor_surface *ds = &bd->ds;
2422    bd->access = paccess;
2423    struct zink_resource *res = zink_descriptor_surface_resource(ds);
2424    VkAccessFlags access = 0;
2425    if (paccess & PIPE_IMAGE_ACCESS_WRITE) {
2426       if (resident) {
2427          res->write_bind_count[0]++;
2428          res->write_bind_count[1]++;
2429       } else {
2430          res->write_bind_count[0]--;
2431          res->write_bind_count[1]--;
2432       }
2433       access |= VK_ACCESS_SHADER_WRITE_BIT;
2434    }
2435    if (paccess & PIPE_IMAGE_ACCESS_READ) {
2436       access |= VK_ACCESS_SHADER_READ_BIT;
2437    }
2438    if (is_buffer)
2439       handle -= ZINK_MAX_BINDLESS_HANDLES;
2440    if (resident) {
2441       update_res_bind_count(ctx, res, false, false);
2442       update_res_bind_count(ctx, res, true, false);
2443       res->image_bind_count[0]++;
2444       res->image_bind_count[1]++;
2445       res->bindless[1]++;
2446       if (is_buffer) {
2447          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
2448             ctx->di.bindless[0].db.buffer_infos[handle].address = res->obj->bda + ds->db.offset;
2449             ctx->di.bindless[0].db.buffer_infos[handle].range = ds->db.size;
2450             ctx->di.bindless[0].db.buffer_infos[handle].format = zink_get_format(zink_screen(ctx->base.screen), ds->db.format);
2451          } else {
2452             if (ds->bufferview->bvci.buffer != res->obj->buffer)
2453                rebind_bindless_bufferview(ctx, res, ds);
2454             VkBufferView *bv = &ctx->di.bindless[1].t.buffer_infos[handle];
2455             *bv = ds->bufferview->buffer_view;
2456          }
2457          zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2458          zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access), true);
2459          if (zink_resource_access_is_write(access))
2460             res->obj->unordered_write = false;
2461          res->obj->unordered_read = false;
2462       } else {
2463          VkDescriptorImageInfo *ii = &ctx->di.bindless[1].img_infos[handle];
2464          ii->sampler = VK_NULL_HANDLE;
2465          ii->imageView = ds->surface->image_view;
2466          ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2467          finalize_image_bind(ctx, res, false);
2468          finalize_image_bind(ctx, res, true);
2469          zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access), false);
2470          res->obj->unordered_write = false;
2471       }
2472       res->gfx_barrier |= VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2473       res->barrier_access[0] |= access;
2474       res->barrier_access[1] |= access;
2475       util_dynarray_append(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2476       uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
2477       util_dynarray_append(&ctx->di.bindless[1].updates, uint32_t, h);
2478    } else {
2479       zero_bindless_descriptor(ctx, handle, is_buffer, true);
2480       util_dynarray_delete_unordered(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
2481       unbind_shader_image_counts(ctx, res, false, false);
2482       unbind_shader_image_counts(ctx, res, true, false);
2483       res->bindless[1]--;
2484       unbind_bindless_descriptor(ctx, res);
2485    }
2486    ctx->di.bindless_dirty[1] = true;
2487 }
2488
2489 static void
2490 zink_set_global_binding(struct pipe_context *pctx,
2491                         unsigned first, unsigned count,
2492                         struct pipe_resource **resources,
2493                         uint32_t **handles)
2494 {
2495    struct zink_context *ctx = zink_context(pctx);
2496
2497    size_t size = ctx->di.global_bindings.capacity;
2498    if (!util_dynarray_resize(&ctx->di.global_bindings, struct pipe_resource*, first + count + 8))
2499       unreachable("zink: out of memory somehow");
2500    if (size != ctx->di.global_bindings.capacity) {
2501       uint8_t *data = ctx->di.global_bindings.data;
2502       memset(data + size, 0, ctx->di.global_bindings.capacity - size);
2503    }
2504
2505    struct pipe_resource **globals = ctx->di.global_bindings.data;
2506    for (unsigned i = 0; i < count; i++) {
2507       if (resources && resources[i]) {
2508          struct zink_resource *res = zink_resource(resources[i]);
2509
2510          util_range_add(&res->base.b, &res->valid_buffer_range, 0, res->base.b.width0);
2511          pipe_resource_reference(&globals[first + i], resources[i]);
2512
2513          uint64_t addr = 0;
2514          memcpy(&addr, handles[i], sizeof(addr));
2515          addr += zink_resource_get_address(zink_screen(pctx->screen), res);
2516          memcpy(handles[i], &addr, sizeof(addr));
2517          zink_resource_usage_set(res, ctx->batch.state, true);
2518          res->obj->unordered_read = res->obj->unordered_write = false;
2519          zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
2520       } else if (globals[i]) {
2521          zink_batch_reference_resource(&ctx->batch, zink_resource(globals[first + i]));
2522          pipe_resource_reference(&globals[first + i], NULL);
2523       }
2524    }
2525 }
2526
2527 static void
2528 zink_set_stencil_ref(struct pipe_context *pctx,
2529                      const struct pipe_stencil_ref ref)
2530 {
2531    struct zink_context *ctx = zink_context(pctx);
2532    ctx->stencil_ref = ref;
2533    ctx->stencil_ref_changed = true;
2534 }
2535
2536 static void
2537 zink_set_clip_state(struct pipe_context *pctx,
2538                     const struct pipe_clip_state *pcs)
2539 {
2540 }
2541
2542 static void
2543 zink_set_tess_state(struct pipe_context *pctx,
2544                     const float default_outer_level[4],
2545                     const float default_inner_level[2])
2546 {
2547    struct zink_context *ctx = zink_context(pctx);
2548    memcpy(&ctx->default_inner_level, default_inner_level, sizeof(ctx->default_inner_level));
2549    memcpy(&ctx->default_outer_level, default_outer_level, sizeof(ctx->default_outer_level));
2550 }
2551
2552 static void
2553 zink_set_patch_vertices(struct pipe_context *pctx, uint8_t patch_vertices)
2554 {
2555    struct zink_context *ctx = zink_context(pctx);
2556    if (zink_set_tcs_key_patches(ctx, patch_vertices)) {
2557       ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = patch_vertices;
2558       if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints)
2559          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, patch_vertices);
2560       else
2561          ctx->gfx_pipeline_state.dirty = true;
2562       zink_flush_dgc_if_enabled(ctx);
2563    }
2564 }
2565
2566 void
2567 zink_update_fbfetch(struct zink_context *ctx)
2568 {
2569    const bool had_fbfetch = ctx->di.fbfetch.imageLayout == VK_IMAGE_LAYOUT_GENERAL;
2570    if (!ctx->gfx_stages[MESA_SHADER_FRAGMENT] ||
2571        !ctx->gfx_stages[MESA_SHADER_FRAGMENT]->info.fs.uses_fbfetch_output) {
2572       if (!had_fbfetch)
2573          return;
2574       ctx->rp_changed = true;
2575       zink_batch_no_rp(ctx);
2576       ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
2577       ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ?
2578                                   VK_NULL_HANDLE :
2579                                   zink_get_dummy_surface(ctx, 0)->image_view;
2580       ctx->invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2581       return;
2582    }
2583
2584    bool changed = !had_fbfetch;
2585    if (ctx->fb_state.cbufs[0]) {
2586       VkImageView fbfetch = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2587       if (!fbfetch)
2588          /* swapchain image: retry later */
2589          return;
2590       changed |= fbfetch != ctx->di.fbfetch.imageView;
2591       ctx->di.fbfetch.imageView = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
2592
2593       bool fbfetch_ms = ctx->fb_state.cbufs[0]->texture->nr_samples > 1;
2594       if (zink_get_fs_base_key(ctx)->fbfetch_ms != fbfetch_ms)
2595          zink_set_fs_base_key(ctx)->fbfetch_ms = fbfetch_ms;
2596    }
2597    ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
2598    if (changed) {
2599       ctx->invalidate_descriptor_state(ctx, MESA_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
2600       if (!had_fbfetch) {
2601          ctx->rp_changed = true;
2602          zink_batch_no_rp(ctx);
2603       }
2604    }
2605 }
2606
2607 void
2608 zink_update_vk_sample_locations(struct zink_context *ctx)
2609 {
2610    if (ctx->gfx_pipeline_state.sample_locations_enabled && ctx->sample_locations_changed) {
2611       unsigned samples = ctx->gfx_pipeline_state.rast_samples + 1;
2612       unsigned idx = util_logbase2_ceil(MAX2(samples, 1));
2613       VkExtent2D grid_size = zink_screen(ctx->base.screen)->maxSampleLocationGridSize[idx];
2614
2615       for (unsigned pixel = 0; pixel < grid_size.width * grid_size.height; pixel++) {
2616          for (unsigned sample = 0; sample < samples; sample++) {
2617             unsigned pixel_x = pixel % grid_size.width;
2618             unsigned pixel_y = pixel / grid_size.width;
2619             unsigned wi = pixel * samples + sample;
2620             unsigned ri = (pixel_y * grid_size.width + pixel_x % grid_size.width);
2621             ri = ri * samples + sample;
2622             ctx->vk_sample_locations[wi].x = (ctx->sample_locations[ri] & 0xf) / 16.0f;
2623             ctx->vk_sample_locations[wi].y = (16 - (ctx->sample_locations[ri] >> 4)) / 16.0f;
2624          }
2625       }
2626    }
2627 }
2628
2629 static unsigned
2630 find_rp_state(struct zink_context *ctx)
2631 {
2632    bool found = false;
2633    /* calc the state idx using the samples to account for msrtss */
2634    unsigned idx = zink_screen(ctx->base.screen)->info.have_EXT_multisampled_render_to_single_sampled && ctx->transient_attachments ?
2635                   util_logbase2_ceil(ctx->gfx_pipeline_state.rast_samples + 1) : 0;
2636    struct set_entry *he = _mesa_set_search_or_add(&ctx->rendering_state_cache[idx], &ctx->gfx_pipeline_state.rendering_info, &found);
2637    struct zink_rendering_info *info;
2638    if (found) {
2639       info = (void*)he->key;
2640       return info->id;
2641    }
2642    info = ralloc(ctx, struct zink_rendering_info);
2643    memcpy(info, &ctx->gfx_pipeline_state.rendering_info, sizeof(VkPipelineRenderingCreateInfo));
2644    info->id = ctx->rendering_state_cache[idx].entries;
2645    he->key = info;
2646    return info->id;
2647 }
2648
2649 unsigned
2650 zink_update_rendering_info(struct zink_context *ctx)
2651 {
2652    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2653       struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2654       ctx->gfx_pipeline_state.rendering_formats[i] = surf ? surf->info.format[0] : VK_FORMAT_UNDEFINED;
2655    }
2656    ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED;
2657    ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = VK_FORMAT_UNDEFINED;
2658    if (ctx->fb_state.zsbuf && zink_is_zsbuf_used(ctx)) {
2659       struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2660       bool has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format));
2661       bool has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format));
2662
2663       if (has_depth)
2664          ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = surf->info.format[0];
2665       if (has_stencil)
2666          ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = surf->info.format[0];
2667    }
2668    return find_rp_state(ctx);
2669 }
2670
2671 static unsigned
2672 calc_max_dummy_fbo_size(struct zink_context *ctx)
2673 {
2674    return MIN2(4096, zink_screen(ctx->base.screen)->info.props.limits.maxImageDimension2D);
2675 }
2676
2677 static unsigned
2678 begin_rendering(struct zink_context *ctx)
2679 {
2680    unsigned clear_buffers = 0;
2681    ctx->gfx_pipeline_state.render_pass = NULL;
2682    zink_update_vk_sample_locations(ctx);
2683    bool has_swapchain = zink_render_update_swapchain(ctx);
2684    if (has_swapchain)
2685       zink_render_fixup_swapchain(ctx);
2686    bool has_depth = false;
2687    bool has_stencil = false;
2688    bool changed_layout = false;
2689    bool changed_size = false;
2690    bool zsbuf_used = zink_is_zsbuf_used(ctx);
2691    bool use_tc_info = !ctx->blitting && ctx->track_renderpasses;
2692    if (ctx->rp_changed || ctx->rp_layout_changed || (!ctx->batch.in_rp && ctx->rp_loadop_changed)) {
2693       /* init imageviews, base loadOp, formats */
2694       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2695          struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2696          if (!surf)
2697             continue;
2698
2699          if (!zink_resource(surf->base.texture)->valid)
2700             ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2701          else
2702             ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2703          if (use_tc_info) {
2704             if (ctx->dynamic_fb.tc_info.cbuf_invalidate & BITFIELD_BIT(i))
2705                ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2706             else
2707                ctx->dynamic_fb.attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2708          }
2709       }
2710
2711       /* unset depth and stencil info: reset below */
2712       VkImageLayout zlayout = ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2713       VkImageLayout slayout = ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED;
2714       ctx->dynamic_fb.info.pDepthAttachment = NULL;
2715       ctx->dynamic_fb.info.pStencilAttachment = NULL;
2716
2717       if (ctx->fb_state.zsbuf && zsbuf_used) {
2718          struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2719          has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format));
2720          has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format));
2721
2722          /* depth may or may not be used but init it anyway */
2723          if (zink_resource(surf->base.texture)->valid)
2724             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
2725          else
2726             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2727
2728          if (use_tc_info) {
2729             if (ctx->dynamic_fb.tc_info.zsbuf_invalidate)
2730                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
2731             else
2732                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
2733          }
2734
2735          /* stencil may or may not be used but init it anyway */
2736          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp;
2737          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].storeOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].storeOp;
2738
2739          if (has_depth) {
2740             ctx->dynamic_fb.info.pDepthAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS];
2741             /* stencil info only set for clears below */
2742          }
2743          if (has_stencil) {
2744             /* must be stencil-only */
2745             ctx->dynamic_fb.info.pStencilAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1];
2746          }
2747       } else {
2748          ctx->dynamic_fb.info.pDepthAttachment = NULL;
2749       }
2750       if (zlayout != (ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2751          changed_layout = true;
2752       if (slayout != (ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED))
2753          changed_layout = true;
2754
2755       /* similar to begin_render_pass(), but just filling in VkRenderingInfo */
2756       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2757          /* these are no-ops */
2758          if (!ctx->fb_state.cbufs[i] || !zink_fb_clear_enabled(ctx, i))
2759             continue;
2760          /* these need actual clear calls inside the rp */
2761          struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(&ctx->fb_clears[i], 0);
2762          if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i])) {
2763             clear_buffers |= (PIPE_CLEAR_COLOR0 << i);
2764             if (zink_fb_clear_count(&ctx->fb_clears[i]) < 2 ||
2765                 zink_fb_clear_element_needs_explicit(clear))
2766                continue;
2767          }
2768          /* we now know there's one clear that can be done here */
2769          memcpy(&ctx->dynamic_fb.attachments[i].clearValue, &clear->color, sizeof(float) * 4);
2770          ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2771       }
2772       if (ctx->fb_state.zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) {
2773          struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
2774          struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0);
2775          if (!zink_fb_clear_element_needs_explicit(clear)) {
2776             /* base zs clear info */
2777             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.depth = clear->zs.depth;
2778             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.stencil = clear->zs.stencil;
2779             /* always init separate stencil attachment */
2780             ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].clearValue.depthStencil.stencil = clear->zs.stencil;
2781             if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_DEPTH))
2782                /* initiate a depth clear */
2783                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2784             if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_STENCIL)) {
2785                /* use a stencil clear, also set stencil attachment */
2786                ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
2787             }
2788          }
2789          if (zink_fb_clear_needs_explicit(fb_clear)) {
2790             for (int j = !zink_fb_clear_element_needs_explicit(clear);
2791                  (clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL && j < zink_fb_clear_count(fb_clear);
2792                  j++)
2793                clear_buffers |= zink_fb_clear_element(fb_clear, j)->zs.bits;
2794          }
2795       }
2796       if (changed_size || changed_layout)
2797          ctx->rp_changed = true;
2798       ctx->rp_loadop_changed = false;
2799       ctx->rp_layout_changed = false;
2800    }
2801
2802    if (!ctx->rp_changed && ctx->batch.in_rp)
2803       return 0;
2804    ctx->rp_changed = false;
2805
2806    /* update pipeline info id for compatibility VUs */
2807    unsigned rp_state = zink_update_rendering_info(ctx);
2808    /* validate zs VUs: attachment must be null or format must be valid */
2809    assert(!ctx->dynamic_fb.info.pDepthAttachment || ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat);
2810    assert(!ctx->dynamic_fb.info.pStencilAttachment || ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat);
2811    bool rp_changed = ctx->gfx_pipeline_state.rp_state != rp_state;
2812    if (!rp_changed && ctx->batch.in_rp)
2813       return 0;
2814
2815    zink_batch_no_rp(ctx);
2816    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2817       VkImageView iv = VK_NULL_HANDLE;
2818       struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2819       if (surf) {
2820          iv = zink_prep_fb_attachment(ctx, surf, i);
2821          if (!iv)
2822             /* dead swapchain */
2823             return 0;
2824       }
2825       ctx->dynamic_fb.attachments[i].imageView = iv;
2826    }
2827    if (has_swapchain) {
2828       ASSERTED struct zink_resource *res = zink_resource(ctx->fb_state.cbufs[0]->texture);
2829       zink_render_fixup_swapchain(ctx);
2830       assert(ctx->dynamic_fb.info.renderArea.extent.width <= res->base.b.width0);
2831       assert(ctx->dynamic_fb.info.renderArea.extent.height <= res->base.b.height0);
2832       assert(ctx->fb_state.width <= res->base.b.width0);
2833       assert(ctx->fb_state.height <= res->base.b.height0);
2834    }
2835    if (ctx->fb_state.zsbuf && zsbuf_used) {
2836       struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2837       VkImageView iv = zink_prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
2838       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageView = iv;
2839       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout = zink_resource(surf->base.texture)->layout;
2840       assert(ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout != VK_IMAGE_LAYOUT_UNDEFINED);
2841       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageView = iv;
2842       ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout = zink_resource(surf->base.texture)->layout;
2843       assert(ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout != VK_IMAGE_LAYOUT_UNDEFINED);
2844       if (ctx->transient_attachments & BITFIELD_BIT(PIPE_MAX_COLOR_BUFS)) {
2845          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveMode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
2846          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1].resolveMode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
2847       } else {
2848          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].resolveMode = 0;
2849          ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1].resolveMode = 0;
2850       }
2851    }
2852    ctx->zsbuf_unused = !zsbuf_used;
2853    assert(ctx->fb_state.width >= ctx->dynamic_fb.info.renderArea.extent.width);
2854    assert(ctx->fb_state.height >= ctx->dynamic_fb.info.renderArea.extent.height);
2855    ctx->gfx_pipeline_state.dirty |= rp_changed;
2856    ctx->gfx_pipeline_state.rp_state = rp_state;
2857
2858    VkMultisampledRenderToSingleSampledInfoEXT msrtss = {
2859       VK_STRUCTURE_TYPE_MULTISAMPLED_RENDER_TO_SINGLE_SAMPLED_INFO_EXT,
2860       NULL,
2861       VK_TRUE,
2862       ctx->gfx_pipeline_state.rast_samples + 1,
2863    };
2864
2865    if (zink_screen(ctx->base.screen)->info.have_EXT_multisampled_render_to_single_sampled)
2866       ctx->dynamic_fb.info.pNext = ctx->transient_attachments ? &msrtss : NULL;
2867    assert(!ctx->transient_attachments || msrtss.rasterizationSamples != VK_SAMPLE_COUNT_1_BIT);
2868    VKCTX(CmdBeginRendering)(ctx->batch.state->cmdbuf, &ctx->dynamic_fb.info);
2869    ctx->batch.in_rp = true;
2870    return clear_buffers;
2871 }
2872
2873 ALWAYS_INLINE static void
2874 update_layered_rendering_state(struct zink_context *ctx)
2875 {
2876    if (!zink_screen(ctx->base.screen)->driver_workarounds.needs_sanitised_layer)
2877       return;
2878    unsigned framebffer_is_layered = zink_framebuffer_get_num_layers(&ctx->fb_state) > 1;
2879    VKCTX(CmdPushConstants)(
2880          ctx->batch.state->cmdbuf,
2881          zink_screen(ctx->base.screen)->gfx_push_constant_layout,
2882          VK_SHADER_STAGE_ALL_GRAPHICS,
2883          offsetof(struct zink_gfx_push_constant, framebuffer_is_layered), sizeof(unsigned),
2884          &framebffer_is_layered);
2885 }
2886
2887 ALWAYS_INLINE static void
2888 batch_ref_fb_surface(struct zink_context *ctx, struct pipe_surface *psurf)
2889 {
2890    if (!psurf)
2891       return;
2892    zink_batch_reference_resource(&ctx->batch, zink_resource(psurf->texture));
2893    struct zink_surface *transient = zink_transient_surface(psurf);
2894    if (transient)
2895       zink_batch_reference_resource(&ctx->batch, zink_resource(transient->base.texture));
2896 }
2897
2898 void
2899 zink_batch_rp(struct zink_context *ctx)
2900 {
2901    assert(!(ctx->batch.in_rp && ctx->rp_changed));
2902    if (!ctx->track_renderpasses && !ctx->blitting) {
2903       if (ctx->rp_tc_info_updated)
2904          zink_parse_tc_info(ctx);
2905    }
2906    if (ctx->batch.in_rp && !ctx->rp_layout_changed)
2907       return;
2908    bool in_rp = ctx->batch.in_rp;
2909    if (!in_rp && ctx->void_clears) {
2910       union pipe_color_union color;
2911       color.f[0] = color.f[1] = color.f[2] = 0;
2912       color.f[3] = 1.0;
2913       ctx->base.clear(&ctx->base, ctx->void_clears, NULL, &color, 0, 0);
2914       ctx->void_clears = 0;
2915    }
2916    if (!ctx->blitting) {
2917       if (ctx->rp_tc_info_updated)
2918          update_tc_info(ctx);
2919       ctx->rp_tc_info_updated = false;
2920    }
2921    bool maybe_has_query_ends = !ctx->track_renderpasses || ctx->dynamic_fb.tc_info.has_query_ends;
2922    ctx->queries_in_rp = maybe_has_query_ends;
2923    /* if possible, out-of-renderpass resume any queries that were stopped when previous rp ended */
2924    if (!ctx->queries_disabled && !maybe_has_query_ends) {
2925       zink_resume_queries(ctx, &ctx->batch);
2926       zink_query_update_gs_states(ctx);
2927    }
2928    unsigned clear_buffers;
2929    /* use renderpass for multisample-to-singlesample or fbfetch:
2930     * - msrtss is TODO
2931     * - dynamic rendering doesn't have input attachments
2932     */
2933    if (!zink_screen(ctx->base.screen)->info.have_KHR_dynamic_rendering ||
2934        (ctx->transient_attachments && !zink_screen(ctx->base.screen)->info.have_EXT_multisampled_render_to_single_sampled) || ctx->fbfetch_outputs)
2935       clear_buffers = zink_begin_render_pass(ctx);
2936    else
2937       clear_buffers = begin_rendering(ctx);
2938    assert(!ctx->rp_changed);
2939    if (!in_rp && ctx->batch.in_rp) {
2940       /* only hit this for valid swapchain and new renderpass */
2941       if (ctx->render_condition.query)
2942          zink_start_conditional_render(ctx);
2943       zink_clear_framebuffer(ctx, clear_buffers);
2944       if (ctx->pipeline_changed[0]) {
2945          for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
2946             batch_ref_fb_surface(ctx, ctx->fb_state.cbufs[i]);
2947          batch_ref_fb_surface(ctx, ctx->fb_state.zsbuf);
2948       }
2949    }
2950    /* unable to previously determine that queries didn't split renderpasses: ensure queries start inside renderpass */
2951    if (!ctx->queries_disabled && maybe_has_query_ends) {
2952       zink_resume_queries(ctx, &ctx->batch);
2953       zink_query_update_gs_states(ctx);
2954    }
2955 }
2956
2957 void
2958 zink_batch_no_rp_safe(struct zink_context *ctx)
2959 {
2960    if (!ctx->batch.in_rp)
2961       return;
2962    zink_flush_dgc_if_enabled(ctx);
2963    if (ctx->render_condition.query)
2964       zink_stop_conditional_render(ctx);
2965    /* suspend all queries that were started in a renderpass
2966     * they can then be resumed upon beginning a new renderpass
2967     */
2968    if (!ctx->queries_disabled)
2969       zink_query_renderpass_suspend(ctx);
2970    if (ctx->gfx_pipeline_state.render_pass)
2971       zink_end_render_pass(ctx);
2972    else {
2973       VKCTX(CmdEndRendering)(ctx->batch.state->cmdbuf);
2974       ctx->batch.in_rp = false;
2975    }
2976    assert(!ctx->batch.in_rp);
2977 }
2978
2979 void
2980 zink_batch_no_rp(struct zink_context *ctx)
2981 {
2982    if (!ctx->batch.in_rp)
2983       return;
2984    if (ctx->track_renderpasses && !ctx->blitting)
2985       tc_renderpass_info_reset(&ctx->dynamic_fb.tc_info);
2986    zink_batch_no_rp_safe(ctx);
2987 }
2988
2989 ALWAYS_INLINE static void
2990 update_res_sampler_layouts(struct zink_context *ctx, struct zink_resource *res)
2991 {
2992    unsigned find = res->sampler_bind_count[0];
2993    for (unsigned i = 0; find && i < MESA_SHADER_COMPUTE; i++) {
2994       u_foreach_bit(slot, res->sampler_binds[i]) {
2995          /* only set layout, skip rest of update */
2996          if (ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i][slot] == res)
2997             ctx->di.textures[i][slot].imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false);
2998          find--;
2999          if (!find) break;
3000       }
3001    }
3002 }
3003
3004 VkImageView
3005 zink_prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i)
3006 {
3007    struct zink_resource *res;
3008    if (!surf) {
3009       surf = zink_get_dummy_surface(ctx, util_logbase2_ceil(ctx->fb_state.samples));
3010       res = zink_resource(surf->base.texture);
3011    } else {
3012       res = zink_resource(surf->base.texture);
3013       zink_batch_resource_usage_set(&ctx->batch, res, true, false);
3014    }
3015
3016    VkAccessFlags access;
3017    VkPipelineStageFlags pipeline;
3018    if (zink_is_swapchain(res)) {
3019       if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
3020          return VK_NULL_HANDLE;
3021       zink_surface_swapchain_update(ctx, surf);
3022       if (!i)
3023          zink_update_fbfetch(ctx);
3024    }
3025    if (ctx->blitting)
3026       return surf->image_view;
3027    VkImageLayout layout;
3028    /* depth attachment is stored as the last attachment, but bitfields always use PIPE_MAX_COLOR_BUFS */
3029    int idx = i == ctx->fb_state.nr_cbufs ? PIPE_MAX_COLOR_BUFS : i;
3030    if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
3031       /* reevaluate feedback loop in case layout change eliminates the loop */
3032       if (!res->sampler_bind_count[0] || (idx == PIPE_MAX_COLOR_BUFS && !zink_is_zsbuf_write(ctx)))
3033          update_feedback_loop_state(ctx, i, ctx->feedback_loops & ~BITFIELD_BIT(idx));
3034    }
3035    if (ctx->track_renderpasses) {
3036       layout = zink_tc_renderpass_info_parse(ctx, &ctx->dynamic_fb.tc_info, idx, &pipeline, &access);
3037       assert(i < ctx->fb_state.nr_cbufs || layout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL || !zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS));
3038       if (i == ctx->fb_state.nr_cbufs && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS))
3039          assert(ctx->dynamic_fb.tc_info.zsbuf_clear || ctx->dynamic_fb.tc_info.zsbuf_clear_partial || ctx->dynamic_fb.tc_info.zsbuf_load);
3040    } else {
3041       if (ctx->gfx_pipeline_state.render_pass) {
3042          layout = zink_render_pass_attachment_get_barrier_info(&ctx->gfx_pipeline_state.render_pass->state.rts[i],
3043                                                                i < ctx->fb_state.nr_cbufs, &pipeline, &access);
3044       } else {
3045          struct zink_rt_attrib rt;
3046          if (i < ctx->fb_state.nr_cbufs)
3047             zink_init_color_attachment(ctx, i, &rt);
3048          else
3049             zink_init_zs_attachment(ctx, &rt);
3050          layout = zink_render_pass_attachment_get_barrier_info(&rt, i < ctx->fb_state.nr_cbufs, &pipeline, &access);
3051          /* avoid unnecessary read-only layout change */
3052          if (layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL &&
3053              res->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL &&
3054              !res->bind_count[0])
3055             layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
3056       }
3057    }
3058    /*
3059       The image subresources for a storage image must be in the VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR or
3060       VK_IMAGE_LAYOUT_GENERAL layout in order to access its data in a shader.
3061       - 14.1.1. Storage Image
3062     */
3063    if (res->image_bind_count[0])
3064       layout = VK_IMAGE_LAYOUT_GENERAL;
3065    else if (!zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_layout &&
3066             layout == VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT)
3067       layout = VK_IMAGE_LAYOUT_GENERAL;
3068    if (res->valid || res->layout != layout)
3069       zink_screen(ctx->base.screen)->image_barrier(ctx, res, layout, access, pipeline);
3070    if (!(res->aspect & VK_IMAGE_ASPECT_COLOR_BIT))
3071       ctx->zsbuf_readonly = res->layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;
3072    res->obj->unordered_read = res->obj->unordered_write = false;
3073    if (i == ctx->fb_state.nr_cbufs && res->sampler_bind_count[0])
3074       update_res_sampler_layouts(ctx, res);
3075    return surf->image_view;
3076 }
3077
3078 static uint32_t
3079 hash_rendering_state(const void *key)
3080 {
3081    const VkPipelineRenderingCreateInfo *info = key;
3082    uint32_t hash = 0;
3083    /*
3084     uint32_t           colorAttachmentCount;
3085     const VkFormat*    pColorAttachmentFormats;
3086     VkFormat           depthAttachmentFormat;
3087     VkFormat           stencilAttachmentFormat;
3088     * this data is not optimally arranged, so it must be manually hashed
3089     */
3090    hash = XXH32(&info->colorAttachmentCount, sizeof(uint32_t), hash);
3091    hash = XXH32(&info->depthAttachmentFormat, sizeof(uint32_t), hash);
3092    hash = XXH32(&info->stencilAttachmentFormat, sizeof(VkFormat), hash);
3093    return XXH32(info->pColorAttachmentFormats, sizeof(VkFormat) * info->colorAttachmentCount, hash);
3094 }
3095
3096 static bool
3097 equals_rendering_state(const void *a, const void *b)
3098 {
3099    const VkPipelineRenderingCreateInfo *ai = a;
3100    const VkPipelineRenderingCreateInfo *bi = b;
3101    return ai->colorAttachmentCount == bi->colorAttachmentCount &&
3102           ai->depthAttachmentFormat == bi->depthAttachmentFormat &&
3103           ai->stencilAttachmentFormat == bi->stencilAttachmentFormat &&
3104           !memcmp(ai->pColorAttachmentFormats, bi->pColorAttachmentFormats, sizeof(VkFormat) * ai->colorAttachmentCount);
3105 }
3106
3107 static uint32_t
3108 hash_framebuffer_imageless(const void *key)
3109 {
3110    struct zink_framebuffer_state* s = (struct zink_framebuffer_state*)key;
3111    return _mesa_hash_data(key, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments);
3112 }
3113
3114 static bool
3115 equals_framebuffer_imageless(const void *a, const void *b)
3116 {
3117    struct zink_framebuffer_state *s = (struct zink_framebuffer_state*)a;
3118    return memcmp(a, b, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments) == 0;
3119 }
3120
3121 void
3122 zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc)
3123 {
3124    struct zink_screen *screen = zink_screen(ctx->base.screen);
3125    unsigned idx = util_logbase2_ceil(MAX2(ctx->gfx_pipeline_state.rast_samples + 1, 1));
3126    loc->sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT;
3127    loc->pNext = NULL;
3128    loc->sampleLocationsPerPixel = 1 << idx;
3129    loc->sampleLocationsCount = ctx->gfx_pipeline_state.rast_samples + 1;
3130    loc->sampleLocationGridSize = screen->maxSampleLocationGridSize[idx];
3131    loc->pSampleLocations = ctx->vk_sample_locations;
3132 }
3133
3134 static void
3135 zink_evaluate_depth_buffer(struct pipe_context *pctx)
3136 {
3137    struct zink_context *ctx = zink_context(pctx);
3138
3139    if (!ctx->fb_state.zsbuf)
3140       return;
3141
3142    struct zink_resource *res = zink_resource(ctx->fb_state.zsbuf->texture);
3143    res->obj->needs_zs_evaluate = true;
3144    zink_init_vk_sample_locations(ctx, &res->obj->zs_evaluate);
3145    zink_batch_no_rp(ctx);
3146 }
3147
3148 static void
3149 sync_flush(struct zink_context *ctx, struct zink_batch_state *bs)
3150 {
3151    if (zink_screen(ctx->base.screen)->threaded_submit)
3152       util_queue_fence_wait(&bs->flush_completed);
3153 }
3154
3155 static inline VkAccessFlags
3156 get_access_flags_for_binding(struct zink_context *ctx, enum zink_descriptor_type type, gl_shader_stage stage, unsigned idx)
3157 {
3158    VkAccessFlags flags = 0;
3159    switch (type) {
3160    case ZINK_DESCRIPTOR_TYPE_UBO:
3161       return VK_ACCESS_UNIFORM_READ_BIT;
3162    case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
3163       return VK_ACCESS_SHADER_READ_BIT;
3164    case ZINK_DESCRIPTOR_TYPE_SSBO: {
3165       flags = VK_ACCESS_SHADER_READ_BIT;
3166       if (ctx->writable_ssbos[stage] & (1 << idx))
3167          flags |= VK_ACCESS_SHADER_WRITE_BIT;
3168       return flags;
3169    }
3170    case ZINK_DESCRIPTOR_TYPE_IMAGE: {
3171       struct zink_image_view *image_view = &ctx->image_views[stage][idx];
3172       if (image_view->base.access & PIPE_IMAGE_ACCESS_READ)
3173          flags |= VK_ACCESS_SHADER_READ_BIT;
3174       if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE)
3175          flags |= VK_ACCESS_SHADER_WRITE_BIT;
3176       return flags;
3177    }
3178    default:
3179       break;
3180    }
3181    unreachable("ACK");
3182    return 0;
3183 }
3184
3185 static void
3186 update_resource_refs_for_stage(struct zink_context *ctx, gl_shader_stage stage)
3187 {
3188    struct zink_batch *batch = &ctx->batch;
3189    unsigned max_slot[] = {
3190       [ZINK_DESCRIPTOR_TYPE_UBO] = ctx->di.num_ubos[stage],
3191       [ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = ctx->di.num_samplers[stage],
3192       [ZINK_DESCRIPTOR_TYPE_SSBO] = ctx->di.num_ssbos[stage],
3193       [ZINK_DESCRIPTOR_TYPE_IMAGE] = ctx->di.num_images[stage]
3194    };
3195    for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) {
3196       for (unsigned j = 0; j < max_slot[i]; j++) {
3197          if (ctx->di.descriptor_res[i][stage][j]) {
3198             struct zink_resource *res = ctx->di.descriptor_res[i][stage][j];
3199             if (!res)
3200                continue;
3201             bool is_buffer = res->obj->is_buffer;
3202             bool is_write = zink_resource_access_is_write(get_access_flags_for_binding(ctx, i, stage, j));
3203             if (zink_is_swapchain(res)) {
3204                if (!zink_kopper_acquire(ctx, res, UINT64_MAX))
3205                   /* technically this is a failure condition, but there's no safe way out */
3206                   continue;
3207             }
3208             zink_batch_resource_usage_set(batch, res, is_write, is_buffer);
3209             if (!ctx->unordered_blitting) {
3210                if (is_write || !res->obj->is_buffer)
3211                   res->obj->unordered_read = res->obj->unordered_write = false;
3212                else
3213                   res->obj->unordered_read = false;
3214             }
3215          }
3216       }
3217    }
3218 }
3219
3220 void
3221 zink_update_descriptor_refs(struct zink_context *ctx, bool compute)
3222 {
3223    struct zink_batch *batch = &ctx->batch;
3224    if (compute) {
3225       update_resource_refs_for_stage(ctx, MESA_SHADER_COMPUTE);
3226       if (ctx->curr_compute)
3227          zink_batch_reference_program(batch, &ctx->curr_compute->base);
3228    } else {
3229       for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++)
3230          update_resource_refs_for_stage(ctx, i);
3231       unsigned vertex_buffers_enabled_mask = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
3232       unsigned last_vbo = util_last_bit(vertex_buffers_enabled_mask);
3233       for (unsigned i = 0; i < last_vbo + 1; i++) {
3234          struct zink_resource *res = zink_resource(ctx->vertex_buffers[i].buffer.resource);
3235          if (res) {
3236             zink_batch_resource_usage_set(batch, res, false, true);
3237             if (!ctx->unordered_blitting)
3238                res->obj->unordered_read = false;
3239          }
3240       }
3241       if (ctx->curr_program)
3242          zink_batch_reference_program(batch, &ctx->curr_program->base);
3243    }
3244    if (ctx->di.bindless_refs_dirty) {
3245       ctx->di.bindless_refs_dirty = false;
3246       for (unsigned i = 0; i < 2; i++) {
3247          util_dynarray_foreach(&ctx->di.bindless[i].resident, struct zink_bindless_descriptor*, bd) {
3248             struct zink_resource *res = zink_descriptor_surface_resource(&(*bd)->ds);
3249             zink_batch_resource_usage_set(&ctx->batch, res, (*bd)->access & PIPE_IMAGE_ACCESS_WRITE, res->obj->is_buffer);
3250             if (!ctx->unordered_blitting) {
3251                if ((*bd)->access & PIPE_IMAGE_ACCESS_WRITE || !res->obj->is_buffer)
3252                   res->obj->unordered_read = res->obj->unordered_write = false;
3253                else
3254                   res->obj->unordered_read = false;
3255             }
3256          }
3257       }
3258    }
3259
3260    unsigned global_count = util_dynarray_num_elements(&ctx->di.global_bindings, struct zink_resource*);
3261    struct zink_resource **globals = ctx->di.global_bindings.data;
3262    for (unsigned i = 0; i < global_count; i++) {
3263       struct zink_resource *res = globals[i];
3264       if (!res)
3265          continue;
3266       zink_batch_resource_usage_set(batch, res, true, true);
3267       res->obj->unordered_read = res->obj->unordered_write = false;
3268    }
3269 }
3270
3271 static void
3272 reapply_color_write(struct zink_context *ctx)
3273 {
3274    struct zink_screen *screen = zink_screen(ctx->base.screen);
3275    assert(screen->info.have_EXT_color_write_enable);
3276    const VkBool32 enables[PIPE_MAX_COLOR_BUFS] = {1, 1, 1, 1, 1, 1, 1, 1};
3277    const VkBool32 disables[PIPE_MAX_COLOR_BUFS] = {0};
3278    const unsigned max_att = MIN2(PIPE_MAX_COLOR_BUFS, screen->info.props.limits.maxColorAttachments);
3279    VKCTX(CmdSetColorWriteEnableEXT)(ctx->batch.state->cmdbuf, max_att, ctx->disable_color_writes ? disables : enables);
3280    VKCTX(CmdSetColorWriteEnableEXT)(ctx->batch.state->barrier_cmdbuf, max_att, enables);
3281    assert(screen->info.have_EXT_extended_dynamic_state);
3282    if (ctx->dsa_state)
3283       VKCTX(CmdSetDepthWriteEnableEXT)(ctx->batch.state->cmdbuf, ctx->disable_color_writes ? VK_FALSE : ctx->dsa_state->hw_state.depth_write);
3284 }
3285
3286 static void
3287 stall(struct zink_context *ctx)
3288 {
3289    struct zink_screen *screen = zink_screen(ctx->base.screen);
3290    sync_flush(ctx, zink_batch_state(ctx->last_fence));
3291    zink_screen_timeline_wait(screen, ctx->last_fence->batch_id, OS_TIMEOUT_INFINITE);
3292    zink_batch_reset_all(ctx);
3293 }
3294
3295 void
3296 zink_reset_ds3_states(struct zink_context *ctx)
3297 {
3298    struct zink_screen *screen = zink_screen(ctx->base.screen);
3299    if (!screen->info.have_EXT_extended_dynamic_state3)
3300       return;
3301    if (screen->have_full_ds3)
3302       ctx->ds3_states = UINT32_MAX;
3303    else
3304       ctx->ds3_states = BITFIELD_MASK(ZINK_DS3_BLEND_A2C);
3305    if (!screen->info.dynamic_state3_feats.extendedDynamicState3AlphaToOneEnable)
3306       ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_BLEND_A21);
3307    if (!screen->info.dynamic_state3_feats.extendedDynamicState3LineStippleEnable)
3308       ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE_ON);
3309    if (screen->driver_workarounds.no_linestipple)
3310       ctx->ds3_states &= ~BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE);
3311 }
3312
3313 static void
3314 flush_batch(struct zink_context *ctx, bool sync)
3315 {
3316    struct zink_batch *batch = &ctx->batch;
3317    assert(!ctx->unordered_blitting);
3318    if (ctx->clears_enabled)
3319       /* start rp to do all the clears */
3320       zink_batch_rp(ctx);
3321    zink_batch_no_rp_safe(ctx);
3322    zink_end_batch(ctx, batch);
3323    ctx->deferred_fence = NULL;
3324
3325    if (sync)
3326       sync_flush(ctx, ctx->batch.state);
3327
3328    if (ctx->batch.state->is_device_lost) {
3329       check_device_lost(ctx);
3330    } else {
3331       struct zink_screen *screen = zink_screen(ctx->base.screen);
3332       zink_start_batch(ctx, batch);
3333       if (screen->info.have_EXT_transform_feedback && ctx->num_so_targets)
3334          ctx->dirty_so_targets = true;
3335       ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
3336       zink_select_draw_vbo(ctx);
3337       zink_select_launch_grid(ctx);
3338
3339       if (ctx->oom_stall)
3340          stall(ctx);
3341       zink_reset_ds3_states(ctx);
3342
3343       ctx->oom_flush = false;
3344       ctx->oom_stall = false;
3345       ctx->dd.bindless_bound = false;
3346       ctx->di.bindless_refs_dirty = true;
3347       ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
3348       if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) {
3349          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch);
3350          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->barrier_cmdbuf, 1);
3351       }
3352       update_feedback_loop_dynamic_state(ctx);
3353       if (screen->info.have_EXT_color_write_enable)
3354          reapply_color_write(ctx);
3355       update_layered_rendering_state(ctx);
3356       tc_renderpass_info_reset(&ctx->dynamic_fb.tc_info);
3357       ctx->rp_tc_info_updated = true;
3358    }
3359 }
3360
3361 void
3362 zink_flush_queue(struct zink_context *ctx)
3363 {
3364    flush_batch(ctx, true);
3365 }
3366
3367 static bool
3368 rebind_fb_surface(struct zink_context *ctx, struct pipe_surface **surf, struct zink_resource *match_res)
3369 {
3370    if (!*surf)
3371       return false;
3372    struct zink_resource *surf_res = zink_resource((*surf)->texture);
3373    if ((match_res == surf_res) || surf_res->obj != zink_csurface(*surf)->obj)
3374       return zink_rebind_ctx_surface(ctx, surf);
3375    return false;
3376 }
3377
3378 static bool
3379 rebind_fb_state(struct zink_context *ctx, struct zink_resource *match_res, bool from_set_fb)
3380 {
3381    bool rebind = false;
3382    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++)
3383       rebind |= rebind_fb_surface(ctx, &ctx->fb_state.cbufs[i], match_res);
3384    rebind |= rebind_fb_surface(ctx, &ctx->fb_state.zsbuf, match_res);
3385    return rebind;
3386 }
3387
3388 static void
3389 unbind_fb_surface(struct zink_context *ctx, struct pipe_surface *surf, unsigned idx, bool changed)
3390 {
3391    ctx->dynamic_fb.attachments[idx].imageView = VK_NULL_HANDLE;
3392    if (!surf)
3393       return;
3394    struct zink_resource *res = zink_resource(surf->texture);
3395    if (changed) {
3396       ctx->rp_changed = true;
3397    }
3398    res->fb_bind_count--;
3399    if (!res->fb_bind_count && !res->bind_count[0])
3400       _mesa_set_remove_key(ctx->need_barriers[0], res);
3401    unsigned feedback_loops = ctx->feedback_loops;
3402    if (ctx->feedback_loops & BITFIELD_BIT(idx)) {
3403       ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
3404       ctx->rp_layout_changed = true;
3405    }
3406    ctx->feedback_loops &= ~BITFIELD_BIT(idx);
3407    if (feedback_loops != ctx->feedback_loops) {
3408       if (idx == PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop_zs) {
3409          if (ctx->gfx_pipeline_state.feedback_loop_zs)
3410             ctx->gfx_pipeline_state.dirty = true;
3411          ctx->gfx_pipeline_state.feedback_loop_zs = false;
3412       } else if (idx < PIPE_MAX_COLOR_BUFS && !zink_screen(ctx->base.screen)->driver_workarounds.always_feedback_loop) {
3413          if (ctx->gfx_pipeline_state.feedback_loop)
3414             ctx->gfx_pipeline_state.dirty = true;
3415          ctx->gfx_pipeline_state.feedback_loop = false;
3416       }
3417    }
3418    res->fb_binds &= ~BITFIELD_BIT(idx);
3419    /* this is called just before the resource loses a reference, so a refcount==1 means the resource will be destroyed */
3420    if (!res->fb_bind_count && res->base.b.reference.count > 1) {
3421       if (ctx->track_renderpasses && !ctx->blitting) {
3422          if (!(res->base.b.bind & PIPE_BIND_DISPLAY_TARGET) && util_format_is_depth_or_stencil(surf->format))
3423             /* assume that all depth buffers which are not swapchain images will be used for sampling to avoid splitting renderpasses */
3424             zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3425          if (!zink_is_swapchain(res) && !util_format_is_depth_or_stencil(surf->format))
3426             /* assume that all color buffers which are not swapchain images will be used for sampling to avoid splitting renderpasses */
3427             zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3428       }
3429       if (res->sampler_bind_count[0]) {
3430          update_res_sampler_layouts(ctx, res);
3431          if (res->layout != VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL && !ctx->blitting)
3432             _mesa_set_add(ctx->need_barriers[0], res);
3433       }
3434    }
3435 }
3436
3437 void
3438 zink_set_null_fs(struct zink_context *ctx)
3439 {
3440    struct zink_screen *screen = zink_screen(ctx->base.screen);
3441    bool prev_disable_fs = ctx->disable_fs;
3442    ctx->disable_fs = ctx->rast_state && ctx->rast_state->base.rasterizer_discard &&
3443                      (ctx->primitives_generated_active || (!ctx->queries_disabled && ctx->primitives_generated_suspended));
3444    struct zink_shader *zs = ctx->gfx_stages[MESA_SHADER_FRAGMENT];
3445    unsigned compact = screen->compact_descriptors ? ZINK_DESCRIPTOR_COMPACT : 0;
3446    /* can't use CWE if side effects */
3447    bool no_cwe = (zs && (zs->ssbos_used || zs->bindless || zs->num_bindings[ZINK_DESCRIPTOR_TYPE_IMAGE - compact])) ||
3448                  ctx->fs_query_active || ctx->occlusion_query_active || !screen->info.have_EXT_color_write_enable;
3449    bool prev_disable_color_writes = ctx->disable_color_writes;
3450    ctx->disable_color_writes = ctx->disable_fs && !no_cwe;
3451
3452    if (ctx->disable_fs == prev_disable_fs) {
3453       /* if this is a true no-op then return */
3454       if (!ctx->disable_fs || ctx->disable_color_writes == !no_cwe)
3455          return;
3456       /* else changing disable modes */
3457    }
3458
3459    /* either of these cases requires removing the previous mode */
3460    if (!ctx->disable_fs || (prev_disable_fs && prev_disable_color_writes != !no_cwe)) {
3461       if (prev_disable_color_writes)
3462          reapply_color_write(ctx);
3463       else
3464          ctx->base.bind_fs_state(&ctx->base, ctx->saved_fs);
3465       ctx->saved_fs = NULL;
3466       /* fs/CWE reenabled, fs active, done */
3467       if (!ctx->disable_fs)
3468          return;
3469    }
3470
3471    /* always use CWE when possible */
3472    if (!no_cwe) {
3473       reapply_color_write(ctx);
3474       return;
3475    }
3476    /* otherwise need to bind a null fs */
3477    if (!ctx->null_fs) {
3478       nir_shader *nir = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, &screen->nir_options, "null_fs").shader;
3479       nir->info.separate_shader = true;
3480       struct pipe_shader_state shstate = {
3481          .type = PIPE_SHADER_IR_NIR,
3482          .tokens = NULL,
3483          .ir.nir = nir
3484       };
3485       ctx->null_fs = ctx->base.create_fs_state(&ctx->base, &shstate);
3486    }
3487    ctx->saved_fs = ctx->gfx_stages[MESA_SHADER_FRAGMENT];
3488    ctx->base.bind_fs_state(&ctx->base, ctx->null_fs);
3489 }
3490
3491 static void
3492 check_framebuffer_surface_mutable(struct pipe_context *pctx, struct pipe_surface *psurf)
3493 {
3494    struct zink_context *ctx = zink_context(pctx);
3495    struct zink_ctx_surface *csurf = (struct zink_ctx_surface *)psurf;
3496    if (!csurf->needs_mutable)
3497       return;
3498    zink_resource_object_init_mutable(ctx, zink_resource(psurf->texture));
3499    struct pipe_surface *psurf2 = pctx->create_surface(pctx, psurf->texture, psurf);
3500    pipe_resource_reference(&psurf2->texture, NULL);
3501    struct zink_ctx_surface *csurf2 = (struct zink_ctx_surface *)psurf2;
3502    zink_surface_reference(zink_screen(pctx->screen), &csurf->surf, csurf2->surf);
3503    pctx->surface_destroy(pctx, psurf2);
3504    csurf->needs_mutable = false;
3505 }
3506
3507 static void
3508 zink_set_framebuffer_state(struct pipe_context *pctx,
3509                            const struct pipe_framebuffer_state *state)
3510 {
3511    struct zink_context *ctx = zink_context(pctx);
3512    struct zink_screen *screen = zink_screen(pctx->screen);
3513    unsigned samples = state->nr_cbufs || state->zsbuf ? 0 : state->samples;
3514    unsigned w = ctx->fb_state.width;
3515    unsigned h = ctx->fb_state.height;
3516    unsigned layers = MAX2(zink_framebuffer_get_num_layers(state), 1);
3517
3518    bool flush_clears = ctx->clears_enabled &&
3519                        (ctx->dynamic_fb.info.layerCount != layers ||
3520                         state->width != w || state->height != h);
3521    if (ctx->clears_enabled && !flush_clears) {
3522       for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3523          if (i >= state->nr_cbufs || !ctx->fb_state.cbufs[i] || !state->cbufs[i])
3524             flush_clears |= zink_fb_clear_enabled(ctx, i);
3525          else if (zink_fb_clear_enabled(ctx, i) && ctx->fb_state.cbufs[i] != state->cbufs[i]) {
3526             struct zink_surface *a = zink_csurface(ctx->fb_state.cbufs[i]);
3527             struct zink_surface *b = zink_csurface(state->cbufs[i]);
3528             if (a == b)
3529                continue;
3530             if (!a || !b || memcmp(&a->base.u.tex, &b->base.u.tex, sizeof(b->base.u.tex)) ||
3531                 a->base.texture != b->base.texture)
3532                flush_clears = true;
3533             else if (a->base.format != b->base.format)
3534                zink_fb_clear_rewrite(ctx, i, a->base.format, b->base.format);
3535          }
3536       }
3537    }
3538    if (ctx->fb_state.zsbuf != state->zsbuf)
3539       flush_clears |= zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS);
3540    if (flush_clears) {
3541       bool queries_disabled = ctx->queries_disabled;
3542       ctx->queries_disabled = true;
3543       zink_batch_rp(ctx);
3544       ctx->queries_disabled = queries_disabled;
3545    }
3546    /* need to ensure we start a new rp on next draw */
3547    zink_batch_no_rp_safe(ctx);
3548    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3549       struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
3550       if (i < state->nr_cbufs)
3551          ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->cbufs[i]);
3552       unbind_fb_surface(ctx, psurf, i, i >= state->nr_cbufs || psurf != state->cbufs[i]);
3553       if (psurf && ctx->needs_present == zink_resource(psurf->texture))
3554          ctx->needs_present = NULL;
3555    }
3556    if (ctx->fb_state.zsbuf) {
3557       struct pipe_surface *psurf = ctx->fb_state.zsbuf;
3558       struct zink_resource *res = zink_resource(psurf->texture);
3559       bool changed = psurf != state->zsbuf;
3560       unbind_fb_surface(ctx, psurf, PIPE_MAX_COLOR_BUFS, changed);
3561       if (!changed)
3562          ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->zsbuf);
3563       if (changed && unlikely(res->obj->needs_zs_evaluate))
3564          /* have to flush zs eval while the sample location data still exists,
3565           * so just throw some random barrier */
3566          zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
3567                                      VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
3568    }
3569    /* renderpass changes if the number or types of attachments change */
3570    ctx->rp_changed |= ctx->fb_state.nr_cbufs != state->nr_cbufs;
3571    ctx->rp_changed |= !!ctx->fb_state.zsbuf != !!state->zsbuf;
3572    if (ctx->fb_state.nr_cbufs != state->nr_cbufs) {
3573       ctx->blend_state_changed |= screen->have_full_ds3;
3574       if (state->nr_cbufs && screen->have_full_ds3)
3575          ctx->ds3_states |= BITFIELD_BIT(ZINK_DS3_BLEND_ON) | BITFIELD_BIT(ZINK_DS3_BLEND_WRITE) | BITFIELD_BIT(ZINK_DS3_BLEND_EQ);
3576    }
3577
3578    util_copy_framebuffer_state(&ctx->fb_state, state);
3579    zink_update_fbfetch(ctx);
3580    ctx->transient_attachments = 0;
3581    ctx->fb_layer_mismatch = 0;
3582
3583    ctx->dynamic_fb.info.renderArea.offset.x = 0;
3584    ctx->dynamic_fb.info.renderArea.offset.y = 0;
3585    ctx->dynamic_fb.info.renderArea.extent.width = state->width;
3586    ctx->dynamic_fb.info.renderArea.extent.height = state->height;
3587    ctx->dynamic_fb.info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
3588    ctx->rp_changed |= ctx->dynamic_fb.info.layerCount != layers;
3589    ctx->dynamic_fb.info.layerCount = layers;
3590    ctx->gfx_pipeline_state.rendering_info.colorAttachmentCount = ctx->fb_state.nr_cbufs;
3591
3592    ctx->void_clears = 0;
3593    for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3594       struct pipe_surface *psurf = ctx->fb_state.cbufs[i];
3595       if (psurf) {
3596          struct zink_surface *transient = zink_transient_surface(psurf);
3597          if (transient || psurf->nr_samples)
3598             ctx->transient_attachments |= BITFIELD_BIT(i);
3599          if (!samples)
3600             samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, psurf->nr_samples ? psurf->nr_samples : 1);
3601          struct zink_resource *res = zink_resource(psurf->texture);
3602          check_framebuffer_surface_mutable(pctx, psurf);
3603          if (zink_csurface(psurf)->info.layerCount > layers)
3604             ctx->fb_layer_mismatch |= BITFIELD_BIT(i);
3605          if (res->modifiers) {
3606             assert(!ctx->needs_present || ctx->needs_present == res);
3607             ctx->needs_present = res;
3608          }
3609          if (res->obj->dt) {
3610             /* #6274 */
3611             if (!zink_screen(ctx->base.screen)->info.have_KHR_swapchain_mutable_format &&
3612                 psurf->format != res->base.b.format) {
3613                static bool warned = false;
3614                if (!warned) {
3615                   mesa_loge("zink: SRGB framebuffer unsupported without KHR_swapchain_mutable_format");
3616                   warned = true;
3617                }
3618             }
3619          }
3620          res->fb_bind_count++;
3621          res->fb_binds |= BITFIELD_BIT(i);
3622          batch_ref_fb_surface(ctx, ctx->fb_state.cbufs[i]);
3623          if (util_format_has_alpha1(psurf->format)) {
3624             if (!res->valid && !zink_fb_clear_full_exists(ctx, i))
3625                ctx->void_clears |= (PIPE_CLEAR_COLOR0 << i);
3626          }
3627       }
3628    }
3629    unsigned depth_bias_scale_factor = ctx->depth_bias_scale_factor;
3630    if (ctx->fb_state.zsbuf) {
3631       struct pipe_surface *psurf = ctx->fb_state.zsbuf;
3632       struct zink_surface *transient = zink_transient_surface(psurf);
3633       check_framebuffer_surface_mutable(pctx, psurf);
3634       batch_ref_fb_surface(ctx, ctx->fb_state.zsbuf);
3635       if (transient || psurf->nr_samples)
3636          ctx->transient_attachments |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3637       if (!samples)
3638          samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, psurf->nr_samples ? psurf->nr_samples : 1);
3639       if (zink_csurface(psurf)->info.layerCount > layers)
3640          ctx->fb_layer_mismatch |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3641       zink_resource(psurf->texture)->fb_bind_count++;
3642       zink_resource(psurf->texture)->fb_binds |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS);
3643       switch (psurf->format) {
3644       case PIPE_FORMAT_Z16_UNORM:
3645       case PIPE_FORMAT_Z16_UNORM_S8_UINT:
3646          ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z16_unscaled_bias;
3647          break;
3648       case PIPE_FORMAT_Z24X8_UNORM:
3649       case PIPE_FORMAT_Z24_UNORM_S8_UINT:
3650       case PIPE_FORMAT_X24S8_UINT:
3651       case PIPE_FORMAT_X8Z24_UNORM:
3652          ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z24_unscaled_bias;
3653          break;
3654       case PIPE_FORMAT_Z32_FLOAT:
3655       case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
3656       case PIPE_FORMAT_Z32_UNORM:
3657          ctx->depth_bias_scale_factor = 1<<23;
3658          break;
3659       default:
3660          ctx->depth_bias_scale_factor = 0;
3661       }
3662    } else {
3663       ctx->depth_bias_scale_factor = 0;
3664    }
3665    if (depth_bias_scale_factor != ctx->depth_bias_scale_factor &&
3666        ctx->rast_state && ctx->rast_state->base.offset_units_unscaled)
3667       ctx->rast_state_changed = true;
3668    rebind_fb_state(ctx, NULL, true);
3669    ctx->fb_state.samples = MAX2(samples, 1);
3670    zink_update_framebuffer_state(ctx);
3671    if (ctx->fb_state.width != w || ctx->fb_state.height != h)
3672       ctx->scissor_changed = true;
3673
3674    uint8_t rast_samples = ctx->fb_state.samples - 1;
3675    if (rast_samples != ctx->gfx_pipeline_state.rast_samples)
3676       zink_update_fs_key_samples(ctx);
3677    if (ctx->gfx_pipeline_state.rast_samples != rast_samples) {
3678       ctx->sample_locations_changed |= ctx->gfx_pipeline_state.sample_locations_enabled;
3679       zink_flush_dgc_if_enabled(ctx);
3680       if (screen->have_full_ds3)
3681          ctx->sample_mask_changed = true;
3682       else
3683          ctx->gfx_pipeline_state.dirty = true;
3684    }
3685    ctx->gfx_pipeline_state.rast_samples = rast_samples;
3686
3687    /* this is an ideal time to oom flush since it won't split a renderpass */
3688    if (ctx->oom_flush && !ctx->unordered_blitting)
3689       flush_batch(ctx, false);
3690    else
3691       update_layered_rendering_state(ctx);
3692
3693    ctx->rp_tc_info_updated = !ctx->blitting;
3694 }
3695
3696 static void
3697 zink_set_blend_color(struct pipe_context *pctx,
3698                      const struct pipe_blend_color *color)
3699 {
3700    struct zink_context *ctx = zink_context(pctx);
3701    memcpy(ctx->blend_constants, color->color, sizeof(float) * 4);
3702    zink_flush_dgc_if_enabled(ctx);
3703 }
3704
3705 static void
3706 zink_set_sample_mask(struct pipe_context *pctx, unsigned sample_mask)
3707 {
3708    struct zink_context *ctx = zink_context(pctx);
3709    if (ctx->gfx_pipeline_state.sample_mask == sample_mask)
3710       return;
3711    ctx->gfx_pipeline_state.sample_mask = sample_mask;
3712    zink_flush_dgc_if_enabled(ctx);
3713    if (zink_screen(pctx->screen)->have_full_ds3)
3714       ctx->sample_mask_changed = true;
3715    else
3716       ctx->gfx_pipeline_state.dirty = true;
3717 }
3718
3719 static void
3720 zink_set_min_samples(struct pipe_context *pctx, unsigned min_samples)
3721 {
3722    struct zink_context *ctx = zink_context(pctx);
3723    ctx->gfx_pipeline_state.min_samples = min_samples - 1;
3724    ctx->gfx_pipeline_state.dirty = true;
3725    zink_flush_dgc_if_enabled(ctx);
3726 }
3727
3728 static void
3729 zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t *locations)
3730 {
3731    struct zink_context *ctx = zink_context(pctx);
3732
3733    ctx->gfx_pipeline_state.sample_locations_enabled = size && locations;
3734    ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
3735    if (size > sizeof(ctx->sample_locations))
3736       size = sizeof(ctx->sample_locations);
3737
3738    if (locations)
3739       memcpy(ctx->sample_locations, locations, size);
3740    zink_flush_dgc_if_enabled(ctx);
3741 }
3742
3743 static void
3744 zink_flush(struct pipe_context *pctx,
3745            struct pipe_fence_handle **pfence,
3746            unsigned flags)
3747 {
3748    struct zink_context *ctx = zink_context(pctx);
3749    bool deferred = flags & PIPE_FLUSH_DEFERRED;
3750    bool deferred_fence = false;
3751    struct zink_batch *batch = &ctx->batch;
3752    struct zink_fence *fence = NULL;
3753    struct zink_screen *screen = zink_screen(ctx->base.screen);
3754    unsigned submit_count = 0;
3755    VkSemaphore export_sem = VK_NULL_HANDLE;
3756
3757    /* triggering clears will force has_work */
3758    if (!deferred && ctx->clears_enabled) {
3759       /* if fbfetch outputs are active, disable them when flushing clears */
3760       unsigned fbfetch_outputs = ctx->fbfetch_outputs;
3761       if (fbfetch_outputs) {
3762          ctx->fbfetch_outputs = 0;
3763          ctx->rp_changed = true;
3764       }
3765       if (ctx->fb_state.zsbuf)
3766          zink_blit_barriers(ctx, NULL, zink_resource(ctx->fb_state.zsbuf->texture), false);
3767
3768       for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3769          if (ctx->fb_state.cbufs[i])
3770             zink_blit_barriers(ctx, NULL, zink_resource(ctx->fb_state.cbufs[i]->texture), false);
3771       }
3772       ctx->blitting = true;
3773       /* start rp to do all the clears */
3774       zink_batch_rp(ctx);
3775       ctx->blitting = false;
3776       ctx->fbfetch_outputs = fbfetch_outputs;
3777       ctx->rp_changed |= fbfetch_outputs > 0;
3778    }
3779
3780    if (flags & PIPE_FLUSH_END_OF_FRAME) {
3781 #ifdef HAVE_RENDERDOC_APP_H
3782       p_atomic_inc(&screen->renderdoc_frame);
3783 #endif
3784       if (ctx->needs_present && ctx->needs_present->obj->dt_idx != UINT32_MAX &&
3785           zink_is_swapchain(ctx->needs_present)) {
3786          zink_kopper_readback_update(ctx, ctx->needs_present);
3787          screen->image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
3788       }
3789       ctx->needs_present = NULL;
3790    }
3791
3792    if (flags & PIPE_FLUSH_FENCE_FD) {
3793       assert(!deferred && pfence);
3794       const VkExportSemaphoreCreateInfo esci = {
3795          .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
3796          .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3797       };
3798       const VkSemaphoreCreateInfo sci = {
3799          .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3800          .pNext = &esci,
3801       };
3802       VkResult result = VKSCR(CreateSemaphore)(screen->dev, &sci, NULL, &export_sem);
3803       if (zink_screen_handle_vkresult(screen, result)) {
3804          assert(!batch->state->signal_semaphore);
3805          batch->state->signal_semaphore = export_sem;
3806          batch->has_work = true;
3807       } else {
3808          mesa_loge("ZINK: vkCreateSemaphore failed (%s)", vk_Result_to_str(result));
3809
3810          /* let flush proceed and ensure a null sem for fence_get_fd to return -1 */
3811          export_sem = VK_NULL_HANDLE;
3812       }
3813    }
3814
3815    if (!batch->has_work) {
3816        if (pfence) {
3817           /* reuse last fence */
3818           fence = ctx->last_fence;
3819        }
3820        if (!deferred) {
3821           struct zink_batch_state *last = zink_batch_state(ctx->last_fence);
3822           if (last) {
3823              sync_flush(ctx, last);
3824              if (last->is_device_lost)
3825                 check_device_lost(ctx);
3826           }
3827        }
3828        if (ctx->tc && !ctx->track_renderpasses)
3829          tc_driver_internal_flush_notify(ctx->tc);
3830    } else {
3831       fence = &batch->state->fence;
3832       submit_count = batch->state->usage.submit_count;
3833       if (deferred && !(flags & PIPE_FLUSH_FENCE_FD) && pfence)
3834          deferred_fence = true;
3835       else
3836          flush_batch(ctx, true);
3837    }
3838
3839    if (pfence) {
3840       struct zink_tc_fence *mfence;
3841
3842       if (flags & TC_FLUSH_ASYNC) {
3843          mfence = zink_tc_fence(*pfence);
3844          assert(mfence);
3845       } else {
3846          mfence = zink_create_tc_fence();
3847
3848          screen->base.fence_reference(&screen->base, pfence, NULL);
3849          *pfence = (struct pipe_fence_handle *)mfence;
3850       }
3851
3852       assert(!mfence->fence);
3853       mfence->fence = fence;
3854       mfence->sem = export_sem;
3855       if (fence) {
3856          mfence->submit_count = submit_count;
3857          util_dynarray_append(&fence->mfences, struct zink_tc_fence *, mfence);
3858       }
3859       if (export_sem) {
3860          pipe_reference(NULL, &mfence->reference);
3861          util_dynarray_append(&ctx->batch.state->fences, struct zink_tc_fence*, mfence);
3862       }
3863
3864       if (deferred_fence) {
3865          assert(fence);
3866          mfence->deferred_ctx = pctx;
3867          assert(!ctx->deferred_fence || ctx->deferred_fence == fence);
3868          ctx->deferred_fence = fence;
3869       }
3870
3871       if (!fence || flags & TC_FLUSH_ASYNC) {
3872          if (!util_queue_fence_is_signalled(&mfence->ready))
3873             util_queue_fence_signal(&mfence->ready);
3874       }
3875    }
3876    if (fence) {
3877       if (!(flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC)))
3878          sync_flush(ctx, zink_batch_state(fence));
3879    }
3880 }
3881
3882 void
3883 zink_fence_wait(struct pipe_context *pctx)
3884 {
3885    struct zink_context *ctx = zink_context(pctx);
3886
3887    if (ctx->batch.has_work)
3888       pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
3889    if (ctx->last_fence)
3890       stall(ctx);
3891 }
3892
3893 void
3894 zink_wait_on_batch(struct zink_context *ctx, uint64_t batch_id)
3895 {
3896    struct zink_batch_state *bs;
3897    if (!batch_id) {
3898       /* not submitted yet */
3899       flush_batch(ctx, true);
3900       bs = zink_batch_state(ctx->last_fence);
3901       assert(bs);
3902       batch_id = bs->fence.batch_id;
3903    }
3904    assert(batch_id);
3905    if (!zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, UINT64_MAX))
3906       check_device_lost(ctx);
3907 }
3908
3909 bool
3910 zink_check_batch_completion(struct zink_context *ctx, uint64_t batch_id)
3911 {
3912    assert(ctx->batch.state);
3913    if (!batch_id)
3914       /* not submitted yet */
3915       return false;
3916
3917    if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id))
3918       return true;
3919
3920    bool success = zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, 0);
3921    if (!success)
3922       check_device_lost(ctx);
3923    return success;
3924 }
3925
3926 static void
3927 zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
3928 {
3929    struct zink_context *ctx = zink_context(pctx);
3930    VkAccessFlags dst = flags == PIPE_TEXTURE_BARRIER_FRAMEBUFFER ?
3931                        VK_ACCESS_INPUT_ATTACHMENT_READ_BIT :
3932                        VK_ACCESS_SHADER_READ_BIT;
3933
3934    if (!ctx->framebuffer || !ctx->framebuffer->state.num_attachments)
3935       return;
3936
3937    /* if this is a fb barrier, flush all pending clears */
3938    if (ctx->rp_clears_enabled && dst == VK_ACCESS_INPUT_ATTACHMENT_READ_BIT)
3939       zink_batch_rp(ctx);
3940
3941    /* this is not an in-renderpass barrier */
3942    if (!ctx->fbfetch_outputs)
3943       zink_batch_no_rp(ctx);
3944
3945    if (zink_screen(ctx->base.screen)->info.have_KHR_synchronization2) {
3946       VkDependencyInfo dep = {0};
3947       dep.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
3948       dep.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
3949       dep.memoryBarrierCount = 1;
3950
3951       VkMemoryBarrier2 dmb = {0};
3952       dmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
3953       dmb.pNext = NULL;
3954       dmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3955       dmb.dstAccessMask = dst;
3956       dmb.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
3957       dmb.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT;
3958       dep.pMemoryBarriers = &dmb;
3959
3960       /* if zs fbfetch is a thing?
3961       if (ctx->fb_state.zsbuf) {
3962          const VkPipelineStageFlagBits2 depth_flags = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
3963          dmb.dstAccessMask |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
3964          dmb.srcStageMask |= depth_flags;
3965          dmb.dstStageMask |= depth_flags;
3966       }
3967       */
3968       VKCTX(CmdPipelineBarrier2)(ctx->batch.state->cmdbuf, &dep);
3969    } else {
3970       VkMemoryBarrier bmb = {0};
3971       bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3972       bmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3973       bmb.dstAccessMask = dst;
3974       VKCTX(CmdPipelineBarrier)(
3975          ctx->batch.state->cmdbuf,
3976          VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
3977          VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
3978          0,
3979          1, &bmb,
3980          0, NULL,
3981          0, NULL
3982       );
3983    }
3984 }
3985
3986 static inline void
3987 mem_barrier(struct zink_context *ctx, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlags src, VkAccessFlags dst)
3988 {
3989    struct zink_batch *batch = &ctx->batch;
3990    VkMemoryBarrier mb;
3991    mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3992    mb.pNext = NULL;
3993    mb.srcAccessMask = src;
3994    mb.dstAccessMask = dst;
3995    zink_batch_no_rp(ctx);
3996    VKCTX(CmdPipelineBarrier)(batch->state->cmdbuf, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL);
3997 }
3998
3999 void
4000 zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute)
4001 {
4002    const VkPipelineStageFlags gfx_flags = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
4003                                           VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
4004                                           VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
4005                                           VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
4006                                           VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
4007    const VkPipelineStageFlags cs_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
4008    VkPipelineStageFlags src = ctx->batch.last_was_compute ? cs_flags : gfx_flags;
4009    VkPipelineStageFlags dst = is_compute ? cs_flags : gfx_flags;
4010
4011    if (ctx->memory_barrier & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_SHADER_BUFFER | PIPE_BARRIER_IMAGE))
4012       mem_barrier(ctx, src, dst, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
4013
4014    if (ctx->memory_barrier & PIPE_BARRIER_CONSTANT_BUFFER)
4015       mem_barrier(ctx, src, dst,
4016                   VK_ACCESS_SHADER_WRITE_BIT,
4017                   VK_ACCESS_UNIFORM_READ_BIT);
4018
4019    if (ctx->memory_barrier & PIPE_BARRIER_INDIRECT_BUFFER)
4020       mem_barrier(ctx, src, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
4021                   VK_ACCESS_SHADER_WRITE_BIT,
4022                   VK_ACCESS_INDIRECT_COMMAND_READ_BIT);
4023    if (!is_compute) {
4024       if (ctx->memory_barrier & PIPE_BARRIER_VERTEX_BUFFER)
4025          mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
4026                      VK_ACCESS_SHADER_WRITE_BIT,
4027                      VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
4028
4029       if (ctx->memory_barrier & PIPE_BARRIER_INDEX_BUFFER)
4030          mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
4031                      VK_ACCESS_SHADER_WRITE_BIT,
4032                      VK_ACCESS_INDEX_READ_BIT);
4033       if (ctx->memory_barrier & PIPE_BARRIER_FRAMEBUFFER)
4034          zink_texture_barrier(&ctx->base, 0);
4035       if (ctx->memory_barrier & PIPE_BARRIER_STREAMOUT_BUFFER)
4036          mem_barrier(ctx, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
4037                             VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
4038                             VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT,
4039                      VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT,
4040                      VK_ACCESS_SHADER_READ_BIT,
4041                      VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT |
4042                      VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT);
4043    }
4044    ctx->memory_barrier = 0;
4045 }
4046
4047 static void
4048 zink_memory_barrier(struct pipe_context *pctx, unsigned flags)
4049 {
4050    struct zink_context *ctx = zink_context(pctx);
4051
4052    flags &= ~PIPE_BARRIER_UPDATE;
4053    if (!flags)
4054       return;
4055
4056    if (flags & PIPE_BARRIER_MAPPED_BUFFER) {
4057       /* TODO: this should flush all persistent buffers in use as I think */
4058       flags &= ~PIPE_BARRIER_MAPPED_BUFFER;
4059    }
4060    ctx->memory_barrier = flags;
4061 }
4062
4063 static void
4064 zink_flush_resource(struct pipe_context *pctx,
4065                     struct pipe_resource *pres)
4066 {
4067    struct zink_context *ctx = zink_context(pctx);
4068    struct zink_resource *res = zink_resource(pres);
4069    if (res->obj->dt) {
4070       if (zink_kopper_acquired(res->obj->dt, res->obj->dt_idx)) {
4071          zink_batch_no_rp_safe(ctx);
4072          zink_kopper_readback_update(ctx, res);
4073          zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
4074          zink_batch_reference_resource_rw(&ctx->batch, res, true);
4075       } else {
4076          ctx->needs_present = res;
4077       }
4078       ctx->batch.swapchain = res;
4079    } else if (res->dmabuf)
4080       res->queue = VK_QUEUE_FAMILY_FOREIGN_EXT;
4081 }
4082
4083 static struct pipe_stream_output_target *
4084 zink_create_stream_output_target(struct pipe_context *pctx,
4085                                  struct pipe_resource *pres,
4086                                  unsigned buffer_offset,
4087                                  unsigned buffer_size)
4088 {
4089    struct zink_so_target *t;
4090    t = CALLOC_STRUCT(zink_so_target);
4091    if (!t)
4092       return NULL;
4093
4094    t->counter_buffer = pipe_buffer_create(pctx->screen, PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_DEFAULT, 4);
4095    if (!t->counter_buffer) {
4096       FREE(t);
4097       return NULL;
4098    }
4099
4100    t->base.reference.count = 1;
4101    t->base.context = pctx;
4102    pipe_resource_reference(&t->base.buffer, pres);
4103    t->base.buffer_offset = buffer_offset;
4104    t->base.buffer_size = buffer_size;
4105
4106    zink_resource(t->base.buffer)->so_valid = true;
4107
4108    return &t->base;
4109 }
4110
4111 static void
4112 zink_stream_output_target_destroy(struct pipe_context *pctx,
4113                                   struct pipe_stream_output_target *psot)
4114 {
4115    struct zink_so_target *t = (struct zink_so_target *)psot;
4116    pipe_resource_reference(&t->counter_buffer, NULL);
4117    pipe_resource_reference(&t->base.buffer, NULL);
4118    FREE(t);
4119 }
4120
4121 static void
4122 zink_set_stream_output_targets(struct pipe_context *pctx,
4123                                unsigned num_targets,
4124                                struct pipe_stream_output_target **targets,
4125                                const unsigned *offsets)
4126 {
4127    struct zink_context *ctx = zink_context(pctx);
4128
4129    /* always set counter_buffer_valid=false on unbind:
4130     * - on resume (indicated by offset==-1), set counter_buffer_valid=true
4131     * - otherwise the counter buffer is invalidated
4132     */
4133
4134    if (num_targets == 0) {
4135       for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4136          if (ctx->so_targets[i]) {
4137             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4138             if (so) {
4139                so->so_bind_count--;
4140                update_res_bind_count(ctx, so, false, true);
4141             }
4142          }
4143          pipe_so_target_reference(&ctx->so_targets[i], NULL);
4144       }
4145       ctx->num_so_targets = 0;
4146    } else {
4147       for (unsigned i = 0; i < num_targets; i++) {
4148          struct zink_so_target *t = zink_so_target(targets[i]);
4149          pipe_so_target_reference(&ctx->so_targets[i], targets[i]);
4150          if (!t)
4151             continue;
4152          if (offsets[0] != (unsigned)-1)
4153             t->counter_buffer_valid = false;
4154          struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4155          if (so) {
4156             so->so_bind_count++;
4157             update_res_bind_count(ctx, so, false, false);
4158          }
4159       }
4160       for (unsigned i = num_targets; i < ctx->num_so_targets; i++) {
4161          if (ctx->so_targets[i]) {
4162             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4163             if (so) {
4164                so->so_bind_count--;
4165                update_res_bind_count(ctx, so, false, true);
4166             }
4167          }
4168          pipe_so_target_reference(&ctx->so_targets[i], NULL);
4169       }
4170       ctx->num_so_targets = num_targets;
4171
4172       /* TODO: possibly avoid rebinding on resume if resuming from same buffers? */
4173       ctx->dirty_so_targets = true;
4174    }
4175    zink_flush_dgc_if_enabled(ctx);
4176 }
4177
4178 void
4179 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res)
4180 {
4181    if (!ctx->framebuffer)
4182       return;
4183    bool did_rebind = false;
4184    if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
4185       for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
4186          if (!ctx->fb_state.cbufs[i] ||
4187              zink_resource(ctx->fb_state.cbufs[i]->texture) != res)
4188             continue;
4189          zink_rebind_ctx_surface(ctx, &ctx->fb_state.cbufs[i]);
4190          did_rebind = true;
4191       }
4192    } else {
4193       if (ctx->fb_state.zsbuf && zink_resource(ctx->fb_state.zsbuf->texture) != res) {
4194          zink_rebind_ctx_surface(ctx, &ctx->fb_state.zsbuf);
4195          did_rebind = true;
4196       }
4197    }
4198
4199    did_rebind |= rebind_fb_state(ctx, res, false);
4200
4201    if (!did_rebind)
4202       return;
4203
4204    zink_batch_no_rp(ctx);
4205    struct zink_framebuffer *fb = zink_get_framebuffer(ctx);
4206    ctx->fb_changed |= ctx->framebuffer != fb;
4207    ctx->framebuffer = fb;
4208 }
4209
4210 ALWAYS_INLINE static struct zink_resource *
4211 rebind_ubo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4212 {
4213    struct zink_resource *res = update_descriptor_state_ubo(ctx, shader, slot,
4214                                                            ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]);
4215    if (res) {
4216       res->obj->unordered_read = false;
4217       res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4218    }
4219    ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, slot, 1);
4220    return res;
4221 }
4222
4223 ALWAYS_INLINE static struct zink_resource *
4224 rebind_ssbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4225 {
4226    const struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4227    struct zink_resource *res = zink_resource(ssbo->buffer);
4228    if (!res)
4229       return NULL;
4230    util_range_add(&res->base.b, &res->valid_buffer_range, ssbo->buffer_offset,
4231                   ssbo->buffer_offset + ssbo->buffer_size);
4232    update_descriptor_state_ssbo(ctx, shader, slot, res);
4233    if (res) {
4234       res->obj->unordered_read = false;
4235       res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4236       if (ctx->writable_ssbos[shader] & BITFIELD_BIT(slot)) {
4237          res->obj->unordered_write = false;
4238          res->obj->access |= VK_ACCESS_SHADER_WRITE_BIT;
4239       }
4240    }
4241    ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SSBO, slot, 1);
4242    return res;
4243 }
4244
4245 ALWAYS_INLINE static struct zink_resource *
4246 rebind_tbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4247 {
4248    struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4249    if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER)
4250       return NULL;
4251    struct zink_resource *res = zink_resource(sampler_view->base.texture);
4252    if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4253       VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci;
4254       bvci.buffer = res->obj->buffer;
4255       zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL);
4256       sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4257    }
4258    update_descriptor_state_sampler(ctx, shader, slot, res);
4259    if (res) {
4260       res->obj->unordered_read = false;
4261       res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4262    }
4263    ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
4264    return res;
4265 }
4266
4267 ALWAYS_INLINE static struct zink_resource *
4268 rebind_ibo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot)
4269 {
4270    struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4271    struct zink_resource *res = zink_resource(image_view->base.resource);
4272    if (!res || res->base.b.target != PIPE_BUFFER)
4273       return NULL;
4274    VkBufferViewCreateInfo bvci;
4275    if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4276       bvci = image_view->buffer_view->bvci;
4277       bvci.buffer = res->obj->buffer;
4278       zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
4279    }
4280    if (!zink_resource_object_init_storage(ctx, res)) {
4281       debug_printf("couldn't create storage image!");
4282       return NULL;
4283    }
4284    if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) {
4285       image_view->buffer_view = get_buffer_view(ctx, res, &bvci);
4286       assert(image_view->buffer_view);
4287    }
4288    if (res) {
4289       res->obj->unordered_read = false;
4290       res->obj->access |= VK_ACCESS_SHADER_READ_BIT;
4291       if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) {
4292          res->obj->unordered_write = false;
4293          res->obj->access |= VK_ACCESS_SHADER_WRITE_BIT;
4294       }
4295    }
4296    util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset,
4297                   image_view->base.u.buf.offset + image_view->base.u.buf.size);
4298    update_descriptor_state_image(ctx, shader, slot, res);
4299    ctx->invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_IMAGE, slot, 1);
4300    return res;
4301 }
4302
4303 static unsigned
4304 rebind_buffer(struct zink_context *ctx, struct zink_resource *res, uint32_t rebind_mask, const unsigned expected_num_rebinds)
4305 {
4306    unsigned num_rebinds = 0;
4307    bool has_write = false;
4308
4309    if (!zink_resource_has_binds(res))
4310       return 0;
4311
4312    assert(!res->bindless[1]); //TODO
4313    if ((rebind_mask & BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER)) || (!rebind_mask && res->so_bind_count && ctx->num_so_targets)) {
4314       for (unsigned i = 0; i < ctx->num_so_targets; i++) {
4315          if (ctx->so_targets[i]) {
4316             struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
4317             if (so && so == res) {
4318                ctx->dirty_so_targets = true;
4319                num_rebinds++;
4320             }
4321          }
4322       }
4323       rebind_mask &= ~BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER);
4324    }
4325    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4326       goto end;
4327
4328    if ((rebind_mask & BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER)) || (!rebind_mask && res->vbo_bind_mask)) {
4329       u_foreach_bit(slot, res->vbo_bind_mask) {
4330          if (ctx->vertex_buffers[slot].buffer.resource != &res->base.b) //wrong context
4331             goto end;
4332          res->obj->access |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT;
4333          res->obj->access_stage |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
4334          res->obj->unordered_read = false;
4335          num_rebinds++;
4336       }
4337       rebind_mask &= ~BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER);
4338       ctx->vertex_buffers_dirty = true;
4339    }
4340    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4341       goto end;
4342
4343    const uint32_t ubo_mask = rebind_mask ?
4344                              rebind_mask & BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES) :
4345                              ((res->ubo_bind_count[0] ? BITFIELD_RANGE(TC_BINDING_UBO_VS, (MESA_SHADER_STAGES - 1)) : 0) |
4346                               (res->ubo_bind_count[1] ? BITFIELD_BIT(TC_BINDING_UBO_CS) : 0));
4347    u_foreach_bit(shader, ubo_mask >> TC_BINDING_UBO_VS) {
4348       u_foreach_bit(slot, res->ubo_bind_mask[shader]) {
4349          if (&res->base.b != ctx->ubos[shader][slot].buffer) //wrong context
4350             goto end;
4351          rebind_ubo(ctx, shader, slot);
4352          num_rebinds++;
4353       }
4354    }
4355    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_UBO_VS, MESA_SHADER_STAGES);
4356    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4357       goto end;
4358
4359    const unsigned ssbo_mask = rebind_mask ?
4360                               rebind_mask & BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES) :
4361                               BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4362    u_foreach_bit(shader, ssbo_mask >> TC_BINDING_SSBO_VS) {
4363       u_foreach_bit(slot, res->ssbo_bind_mask[shader]) {
4364          struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
4365          if (&res->base.b != ssbo->buffer) //wrong context
4366             goto end;
4367          rebind_ssbo(ctx, shader, slot);
4368          has_write |= (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0;
4369          num_rebinds++;
4370       }
4371    }
4372    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SSBO_VS, MESA_SHADER_STAGES);
4373    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4374       goto end;
4375    const unsigned sampler_mask = rebind_mask ?
4376                                  rebind_mask & BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES) :
4377                                  BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4378    u_foreach_bit(shader, sampler_mask >> TC_BINDING_SAMPLERVIEW_VS) {
4379       u_foreach_bit(slot, res->sampler_binds[shader]) {
4380          struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
4381          if (&res->base.b != sampler_view->base.texture) //wrong context
4382             goto end;
4383          rebind_tbo(ctx, shader, slot);
4384          num_rebinds++;
4385       }
4386    }
4387    rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, MESA_SHADER_STAGES);
4388    if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
4389       goto end;
4390
4391    const unsigned image_mask = rebind_mask ?
4392                                rebind_mask & BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES) :
4393                                BITFIELD_RANGE(TC_BINDING_IMAGE_VS, MESA_SHADER_STAGES);
4394    unsigned num_image_rebinds_remaining = rebind_mask ? expected_num_rebinds - num_rebinds : res->image_bind_count[0] + res->image_bind_count[1];
4395    u_foreach_bit(shader, image_mask >> TC_BINDING_IMAGE_VS) {
4396       for (unsigned slot = 0; num_image_rebinds_remaining && slot < ctx->di.num_images[shader]; slot++) {
4397          struct zink_resource *cres = ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_IMAGE][shader][slot];
4398          if (res != cres)
4399             continue;
4400
4401          rebind_ibo(ctx, shader, slot);
4402          const struct zink_image_view *image_view = &ctx->image_views[shader][slot];
4403          has_write |= (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) != 0;
4404          num_image_rebinds_remaining--;
4405          num_rebinds++;
4406       }
4407    }
4408 end:
4409    if (num_rebinds)
4410       zink_batch_resource_usage_set(&ctx->batch, res, has_write, true);
4411    return num_rebinds;
4412 }
4413
4414 void
4415 zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4416                  unsigned dst_offset, unsigned src_offset, unsigned size)
4417 {
4418    VkBufferCopy region;
4419    region.srcOffset = src_offset;
4420    region.dstOffset = dst_offset;
4421    region.size = size;
4422
4423    struct zink_batch *batch = &ctx->batch;
4424
4425    struct pipe_box box = {(int)src_offset, 0, 0, (int)size, 0, 0};
4426    /* must barrier if something wrote the valid buffer range */
4427    bool valid_write = zink_check_valid_buffer_src_access(ctx, src, src_offset, size);
4428    bool unordered_src = !valid_write && !zink_check_unordered_transfer_access(src, 0, &box);
4429    zink_screen(ctx->base.screen)->buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, 0);
4430    bool unordered_dst = zink_resource_buffer_transfer_dst_barrier(ctx, dst, dst_offset, size);
4431    bool can_unorder = unordered_dst && unordered_src && !(zink_debug & ZINK_DEBUG_NOREORDER);
4432    VkCommandBuffer cmdbuf = can_unorder ? ctx->batch.state->barrier_cmdbuf : zink_get_cmdbuf(ctx, src, dst);
4433    ctx->batch.state->has_barriers |= can_unorder;
4434    zink_batch_reference_resource_rw(batch, src, false);
4435    zink_batch_reference_resource_rw(batch, dst, true);
4436    if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4437       VkMemoryBarrier mb;
4438       mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4439       mb.pNext = NULL;
4440       mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4441       mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4442       VKCTX(CmdPipelineBarrier)(cmdbuf,
4443                                 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4444                                 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4445                                 0, 1, &mb, 0, NULL, 0, NULL);
4446    }
4447    bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_buffer(%d)", size);
4448    VKCTX(CmdCopyBuffer)(cmdbuf, src->obj->buffer, dst->obj->buffer, 1, &region);
4449    zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4450 }
4451
4452 void
4453 zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
4454                        unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4455                        unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags)
4456 {
4457    struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst;
4458    struct zink_resource *use_img = img;
4459    struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src;
4460    struct zink_batch *batch = &ctx->batch;
4461    bool needs_present_readback = false;
4462
4463    bool buf2img = buf == src;
4464
4465    if (buf2img) {
4466       if (zink_is_swapchain(img)) {
4467          if (!zink_kopper_acquire(ctx, img, UINT64_MAX))
4468             return;
4469       }
4470       struct pipe_box box = *src_box;
4471       box.x = dstx;
4472       box.y = dsty;
4473       box.z = dstz;
4474       zink_resource_image_transfer_dst_barrier(ctx, img, dst_level, &box);
4475       zink_screen(ctx->base.screen)->buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
4476    } else {
4477       if (zink_is_swapchain(img))
4478          needs_present_readback = zink_kopper_acquire_readback(ctx, img, &use_img);
4479       zink_screen(ctx->base.screen)->image_barrier(ctx, use_img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0);
4480       zink_resource_buffer_transfer_dst_barrier(ctx, buf, dstx, src_box->width);
4481    }
4482
4483    VkBufferImageCopy region = {0};
4484    region.bufferOffset = buf2img ? src_box->x : dstx;
4485    region.bufferRowLength = 0;
4486    region.bufferImageHeight = 0;
4487    region.imageSubresource.mipLevel = buf2img ? dst_level : src_level;
4488    enum pipe_texture_target img_target = img->base.b.target;
4489    if (img->need_2D)
4490       img_target = img_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4491    switch (img_target) {
4492    case PIPE_TEXTURE_CUBE:
4493    case PIPE_TEXTURE_CUBE_ARRAY:
4494    case PIPE_TEXTURE_2D_ARRAY:
4495    case PIPE_TEXTURE_1D_ARRAY:
4496       /* these use layer */
4497       region.imageSubresource.baseArrayLayer = buf2img ? dstz : src_box->z;
4498       region.imageSubresource.layerCount = src_box->depth;
4499       region.imageOffset.z = 0;
4500       region.imageExtent.depth = 1;
4501       break;
4502    case PIPE_TEXTURE_3D:
4503       /* this uses depth */
4504       region.imageSubresource.baseArrayLayer = 0;
4505       region.imageSubresource.layerCount = 1;
4506       region.imageOffset.z = buf2img ? dstz : src_box->z;
4507       region.imageExtent.depth = src_box->depth;
4508       break;
4509    default:
4510       /* these must only copy one layer */
4511       region.imageSubresource.baseArrayLayer = 0;
4512       region.imageSubresource.layerCount = 1;
4513       region.imageOffset.z = 0;
4514       region.imageExtent.depth = 1;
4515    }
4516    region.imageOffset.x = buf2img ? dstx : src_box->x;
4517    region.imageOffset.y = buf2img ? dsty : src_box->y;
4518
4519    region.imageExtent.width = src_box->width;
4520    region.imageExtent.height = src_box->height;
4521
4522    /* never promote to unordered if swapchain was acquired */
4523    VkCommandBuffer cmdbuf = needs_present_readback ?
4524                             ctx->batch.state->cmdbuf :
4525                             buf2img ? zink_get_cmdbuf(ctx, buf, use_img) : zink_get_cmdbuf(ctx, use_img, buf);
4526    zink_batch_reference_resource_rw(batch, use_img, buf2img);
4527    zink_batch_reference_resource_rw(batch, buf, !buf2img);
4528
4529    /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage
4530     * to indicate whether to copy either the depth or stencil aspects
4531     */
4532    unsigned aspects = 0;
4533    if (map_flags) {
4534       assert((map_flags & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) !=
4535              (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY));
4536       if (map_flags & PIPE_MAP_DEPTH_ONLY)
4537          aspects = VK_IMAGE_ASPECT_DEPTH_BIT;
4538       else if (map_flags & PIPE_MAP_STENCIL_ONLY)
4539          aspects = VK_IMAGE_ASPECT_STENCIL_BIT;
4540    }
4541    if (!aspects)
4542       aspects = img->aspect;
4543    if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4544       VkMemoryBarrier mb;
4545       mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4546       mb.pNext = NULL;
4547       mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4548       mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4549       VKCTX(CmdPipelineBarrier)(cmdbuf,
4550                                 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4551                                 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4552                                 0, 1, &mb, 0, NULL, 0, NULL);
4553    }
4554    while (aspects) {
4555       int aspect = 1 << u_bit_scan(&aspects);
4556       region.imageSubresource.aspectMask = aspect;
4557
4558       /* MSAA transfers should have already been handled by U_TRANSFER_HELPER_MSAA_MAP, since
4559        * there's no way to resolve using this interface:
4560        *
4561        * srcImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4562        * - vkCmdCopyImageToBuffer spec
4563        *
4564        * dstImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
4565        * - vkCmdCopyBufferToImage spec
4566        */
4567       assert(img->base.b.nr_samples <= 1);
4568       bool marker;
4569       if (buf2img) {
4570          marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_buffer2image(%s, %dx%dx%d)",
4571                                                    util_format_short_name(dst->base.b.format),
4572                                                    region.imageExtent.width,
4573                                                    region.imageExtent.height,
4574                                                    MAX2(region.imageSubresource.layerCount, region.imageExtent.depth));
4575          VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, use_img->obj->image, use_img->layout, 1, &region);
4576       } else {
4577          marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_image2buffer(%s, %dx%dx%d)",
4578                                                    util_format_short_name(src->base.b.format),
4579                                                    region.imageExtent.width,
4580                                                    region.imageExtent.height,
4581                                                    MAX2(region.imageSubresource.layerCount, region.imageExtent.depth));
4582          VKCTX(CmdCopyImageToBuffer)(cmdbuf, use_img->obj->image, use_img->layout, buf->obj->buffer, 1, &region);
4583       }
4584       zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4585    }
4586    if (needs_present_readback) {
4587       if (buf2img) {
4588          img->obj->unordered_write = false;
4589          buf->obj->unordered_read = false;
4590       } else {
4591          img->obj->unordered_read = false;
4592          buf->obj->unordered_write = false;
4593       }
4594       zink_kopper_present_readback(ctx, img);
4595    }
4596
4597    if (ctx->oom_flush && !ctx->batch.in_rp && !ctx->unordered_blitting)
4598       flush_batch(ctx, false);
4599 }
4600
4601 static void
4602 zink_resource_copy_region(struct pipe_context *pctx,
4603                           struct pipe_resource *pdst,
4604                           unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
4605                           struct pipe_resource *psrc,
4606                           unsigned src_level, const struct pipe_box *src_box)
4607 {
4608    struct zink_resource *dst = zink_resource(pdst);
4609    struct zink_resource *src = zink_resource(psrc);
4610    struct zink_context *ctx = zink_context(pctx);
4611    if (dst->base.b.target != PIPE_BUFFER && src->base.b.target != PIPE_BUFFER) {
4612       VkImageCopy region;
4613       /* fill struct holes */
4614       memset(&region, 0, sizeof(region));
4615       if (util_format_get_num_planes(src->base.b.format) == 1 &&
4616           util_format_get_num_planes(dst->base.b.format) == 1) {
4617       /* If neither the calling command’s srcImage nor the calling command’s dstImage
4618        * has a multi-planar image format then the aspectMask member of srcSubresource
4619        * and dstSubresource must match
4620        *
4621        * -VkImageCopy spec
4622        */
4623          assert(src->aspect == dst->aspect);
4624       } else
4625          unreachable("planar formats not yet handled");
4626
4627
4628       region.srcSubresource.aspectMask = src->aspect;
4629       region.srcSubresource.mipLevel = src_level;
4630       enum pipe_texture_target src_target = src->base.b.target;
4631       if (src->need_2D)
4632          src_target = src_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4633       switch (src_target) {
4634       case PIPE_TEXTURE_CUBE:
4635       case PIPE_TEXTURE_CUBE_ARRAY:
4636       case PIPE_TEXTURE_2D_ARRAY:
4637       case PIPE_TEXTURE_1D_ARRAY:
4638          /* these use layer */
4639          region.srcSubresource.baseArrayLayer = src_box->z;
4640          region.srcSubresource.layerCount = src_box->depth;
4641          region.srcOffset.z = 0;
4642          region.extent.depth = 1;
4643          break;
4644       case PIPE_TEXTURE_3D:
4645          /* this uses depth */
4646          region.srcSubresource.baseArrayLayer = 0;
4647          region.srcSubresource.layerCount = 1;
4648          region.srcOffset.z = src_box->z;
4649          region.extent.depth = src_box->depth;
4650          break;
4651       default:
4652          /* these must only copy one layer */
4653          region.srcSubresource.baseArrayLayer = 0;
4654          region.srcSubresource.layerCount = 1;
4655          region.srcOffset.z = 0;
4656          region.extent.depth = 1;
4657       }
4658
4659       region.srcOffset.x = src_box->x;
4660       region.srcOffset.y = src_box->y;
4661
4662       region.dstSubresource.aspectMask = dst->aspect;
4663       region.dstSubresource.mipLevel = dst_level;
4664       enum pipe_texture_target dst_target = dst->base.b.target;
4665       if (dst->need_2D)
4666          dst_target = dst_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY;
4667       switch (dst_target) {
4668       case PIPE_TEXTURE_CUBE:
4669       case PIPE_TEXTURE_CUBE_ARRAY:
4670       case PIPE_TEXTURE_2D_ARRAY:
4671       case PIPE_TEXTURE_1D_ARRAY:
4672          /* these use layer */
4673          region.dstSubresource.baseArrayLayer = dstz;
4674          region.dstSubresource.layerCount = src_box->depth;
4675          region.dstOffset.z = 0;
4676          break;
4677       case PIPE_TEXTURE_3D:
4678          /* this uses depth */
4679          region.dstSubresource.baseArrayLayer = 0;
4680          region.dstSubresource.layerCount = 1;
4681          region.dstOffset.z = dstz;
4682          break;
4683       default:
4684          /* these must only copy one layer */
4685          region.dstSubresource.baseArrayLayer = 0;
4686          region.dstSubresource.layerCount = 1;
4687          region.dstOffset.z = 0;
4688       }
4689
4690       region.dstOffset.x = dstx;
4691       region.dstOffset.y = dsty;
4692       region.extent.width = src_box->width;
4693       region.extent.height = src_box->height;
4694
4695       /* ignore no-op copies */
4696       if (src == dst &&
4697           !memcmp(&region.dstOffset, &region.srcOffset, sizeof(region.srcOffset)) &&
4698           !memcmp(&region.dstSubresource, &region.srcSubresource, sizeof(region.srcSubresource)))
4699          return;
4700
4701       zink_fb_clears_apply_or_discard(ctx, pdst, (struct u_rect){dstx, dstx + src_box->width, dsty, dsty + src_box->height}, false);
4702       zink_fb_clears_apply_region(ctx, psrc, zink_rect_from_box(src_box));
4703
4704       struct zink_batch *batch = &ctx->batch;
4705       zink_resource_setup_transfer_layouts(ctx, src, dst);
4706       VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst);
4707       zink_batch_reference_resource_rw(batch, src, false);
4708       zink_batch_reference_resource_rw(batch, dst, true);
4709
4710       if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
4711          VkMemoryBarrier mb;
4712          mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
4713          mb.pNext = NULL;
4714          mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
4715          mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
4716          VKCTX(CmdPipelineBarrier)(cmdbuf,
4717                                    VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4718                                    VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
4719                                    0, 1, &mb, 0, NULL, 0, NULL);
4720       }
4721       bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "copy_image(%s->%s, %dx%dx%d)",
4722                                                 util_format_short_name(psrc->format),
4723                                                 util_format_short_name(pdst->format),
4724                                                 region.extent.width,
4725                                                 region.extent.height,
4726                                                 MAX2(region.srcSubresource.layerCount, region.extent.depth));
4727       VKCTX(CmdCopyImage)(cmdbuf, src->obj->image, src->layout,
4728                      dst->obj->image, dst->layout,
4729                      1, &region);
4730       zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
4731    } else if (dst->base.b.target == PIPE_BUFFER &&
4732               src->base.b.target == PIPE_BUFFER) {
4733       zink_copy_buffer(ctx, dst, src, dstx, src_box->x, src_box->width);
4734    } else
4735       zink_copy_image_buffer(ctx, dst, src, dst_level, dstx, dsty, dstz, src_level, src_box, 0);
4736    if (ctx->oom_flush && !ctx->batch.in_rp && !ctx->unordered_blitting)
4737       flush_batch(ctx, false);
4738 }
4739
4740 static bool
4741 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
4742 {
4743    struct zink_context *ctx = zink_context(pctx);
4744    struct zink_resource *res = zink_resource(pres);
4745    struct zink_screen *screen = zink_screen(pctx->screen);
4746
4747    /* if any current usage exists, flush the queue */
4748    if (zink_resource_has_unflushed_usage(res))
4749       zink_flush_queue(ctx);
4750
4751    VkSemaphore sem = VK_NULL_HANDLE;
4752    bool ret = zink_bo_commit(screen, res, level, box, commit, &sem);
4753    if (ret) {
4754       if (sem)
4755          zink_batch_add_wait_semaphore(&ctx->batch, sem);
4756    } else {
4757       check_device_lost(ctx);
4758    }
4759
4760    return ret;
4761 }
4762
4763 static void
4764 rebind_image(struct zink_context *ctx, struct zink_resource *res)
4765 {
4766    assert(!ctx->blitting);
4767    if (res->fb_binds)
4768       zink_rebind_framebuffer(ctx, res);
4769    if (!zink_resource_has_binds(res))
4770       return;
4771    for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
4772       if (res->sampler_binds[i]) {
4773          for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
4774             struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
4775             if (sv && sv->base.texture == &res->base.b) {
4776                struct pipe_surface *psurf = &sv->image_view->base;
4777                zink_rebind_surface(ctx, &psurf);
4778                sv->image_view = zink_surface(psurf);
4779                ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
4780                update_descriptor_state_sampler(ctx, i, j, res);
4781             }
4782          }
4783       }
4784       if (!res->image_bind_count[i == MESA_SHADER_COMPUTE])
4785          continue;
4786       for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
4787          if (zink_resource(ctx->image_views[i][j].base.resource) == res) {
4788             ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
4789             update_descriptor_state_image(ctx, i, j, res);
4790             _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
4791          }
4792       }
4793    }
4794 }
4795
4796 bool
4797 zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res)
4798 {
4799    if (res->base.b.target == PIPE_BUFFER) {
4800       /* force counter buffer reset */
4801       res->so_valid = false;
4802       return rebind_buffer(ctx, res, 0, 0) == res->bind_count[0] + res->bind_count[1];
4803    }
4804    rebind_image(ctx, res);
4805    return false;
4806 }
4807
4808 void
4809 zink_rebind_all_buffers(struct zink_context *ctx)
4810 {
4811    struct zink_batch *batch = &ctx->batch;
4812    ctx->vertex_buffers_dirty = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask > 0;
4813    ctx->dirty_so_targets = ctx->num_so_targets > 0;
4814    if (ctx->num_so_targets)
4815       zink_screen(ctx->base.screen)->buffer_barrier(ctx, zink_resource(ctx->dummy_xfb_buffer),
4816                                    VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
4817    for (unsigned shader = MESA_SHADER_VERTEX; shader < MESA_SHADER_STAGES; shader++) {
4818       for (unsigned slot = 0; slot < ctx->di.num_ubos[shader]; slot++) {
4819          struct zink_resource *res = rebind_ubo(ctx, shader, slot);
4820          if (res)
4821             zink_batch_resource_usage_set(batch, res, false, true);
4822       }
4823       for (unsigned slot = 0; slot < ctx->di.num_sampler_views[shader]; slot++) {
4824          struct zink_resource *res = rebind_tbo(ctx, shader, slot);
4825          if (res)
4826             zink_batch_resource_usage_set(batch, res, false, true);
4827       }
4828       for (unsigned slot = 0; slot < ctx->di.num_ssbos[shader]; slot++) {
4829          struct zink_resource *res = rebind_ssbo(ctx, shader, slot);
4830          if (res)
4831             zink_batch_resource_usage_set(batch, res, (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0, true);
4832       }
4833       for (unsigned slot = 0; slot < ctx->di.num_images[shader]; slot++) {
4834          struct zink_resource *res = rebind_ibo(ctx, shader, slot);
4835          if (res)
4836             zink_batch_resource_usage_set(batch, res, (ctx->image_views[shader][slot].base.access & PIPE_IMAGE_ACCESS_WRITE) != 0, true);
4837       }
4838    }
4839 }
4840
4841 void
4842 zink_rebind_all_images(struct zink_context *ctx)
4843 {
4844    assert(!ctx->blitting);
4845    rebind_fb_state(ctx, NULL, false);
4846     for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
4847       for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
4848          struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
4849          if (!sv || sv->image_view->base.texture->target == PIPE_BUFFER || !sv->image_view)
4850             continue;
4851          struct zink_resource *res = zink_resource(sv->image_view->base.texture);
4852          if (res->obj != sv->image_view->obj) {
4853              struct pipe_surface *psurf = &sv->image_view->base;
4854              zink_rebind_surface(ctx, &psurf);
4855              sv->image_view = zink_surface(psurf);
4856              ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
4857              update_descriptor_state_sampler(ctx, i, j, res);
4858          }
4859       }
4860       for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
4861          struct zink_image_view *image_view = &ctx->image_views[i][j];
4862          struct zink_resource *res = zink_resource(image_view->base.resource);
4863          if (!res || res->base.b.target == PIPE_BUFFER)
4864             continue;
4865          if (ctx->image_views[i][j].surface->obj != res->obj) {
4866             zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
4867             image_view->surface = create_image_surface(ctx, &image_view->base, i == MESA_SHADER_COMPUTE);
4868             ctx->invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
4869             update_descriptor_state_image(ctx, i, j, res);
4870             _mesa_set_add(ctx->need_barriers[i == MESA_SHADER_COMPUTE], res);
4871          }
4872       }
4873    }
4874 }
4875
4876 static void
4877 zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *dst,
4878                                     struct pipe_resource *src, unsigned num_rebinds,
4879                                     uint32_t rebind_mask, uint32_t delete_buffer_id)
4880 {
4881    struct zink_resource *d = zink_resource(dst);
4882    struct zink_resource *s = zink_resource(src);
4883    struct zink_context *ctx = zink_context(pctx);
4884    struct zink_screen *screen = zink_screen(pctx->screen);
4885
4886    assert(d->internal_format == s->internal_format);
4887    assert(d->obj);
4888    assert(s->obj);
4889    util_idalloc_mt_free(&screen->buffer_ids, delete_buffer_id);
4890    zink_batch_reference_resource(&ctx->batch, d);
4891    /* don't be too creative */
4892    zink_resource_object_reference(screen, &d->obj, s->obj);
4893    d->valid_buffer_range = s->valid_buffer_range;
4894    zink_resource_copies_reset(d);
4895    /* force counter buffer reset */
4896    d->so_valid = false;
4897    if (num_rebinds && rebind_buffer(ctx, d, rebind_mask, num_rebinds) < num_rebinds)
4898       ctx->buffer_rebind_counter = p_atomic_inc_return(&screen->buffer_rebind_counter);
4899 }
4900
4901 static bool
4902 zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage)
4903 {
4904    struct zink_screen *screen = zink_screen(pscreen);
4905    struct zink_resource *res = zink_resource(pres);
4906    uint32_t check_usage = 0;
4907    if (usage & PIPE_MAP_READ)
4908       check_usage |= ZINK_RESOURCE_ACCESS_WRITE;
4909    if (usage & PIPE_MAP_WRITE)
4910       check_usage |= ZINK_RESOURCE_ACCESS_RW;
4911    return !zink_resource_usage_check_completion(screen, res, check_usage);
4912 }
4913
4914 static void
4915 zink_emit_string_marker(struct pipe_context *pctx,
4916                         const char *string, int len)
4917 {
4918    struct zink_screen *screen = zink_screen(pctx->screen);
4919    struct zink_batch *batch = &zink_context(pctx)->batch;
4920
4921    /* make sure string is nul-terminated */
4922    char buf[512], *temp = NULL;
4923    if (len < ARRAY_SIZE(buf)) {
4924       memcpy(buf, string, len);
4925       buf[len] = '\0';
4926       string = buf;
4927    } else
4928       string = temp = strndup(string, len);
4929
4930    VkDebugUtilsLabelEXT label = {
4931       VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, NULL,
4932       string,
4933       { 0 }
4934    };
4935    screen->vk.CmdInsertDebugUtilsLabelEXT(batch->state->cmdbuf, &label);
4936    free(temp);
4937 }
4938
4939 VkIndirectCommandsLayoutTokenNV *
4940 zink_dgc_add_token(struct zink_context *ctx, VkIndirectCommandsTokenTypeNV type, void **mem)
4941 {
4942    size_t size = 0;
4943    struct zink_screen *screen = zink_screen(ctx->base.screen);
4944    VkIndirectCommandsLayoutTokenNV *ret = util_dynarray_grow(&ctx->dgc.tokens, VkIndirectCommandsLayoutTokenNV, 1);
4945    ret->sType = VK_STRUCTURE_TYPE_INDIRECT_COMMANDS_LAYOUT_TOKEN_NV;
4946    ret->pNext = NULL;
4947    ret->tokenType = type;
4948    ret->vertexDynamicStride = ctx->gfx_pipeline_state.uses_dynamic_stride;
4949    ret->indirectStateFlags = 0;
4950    ret->indexTypeCount = 0;
4951    switch (type) {
4952    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_VERTEX_BUFFER_NV:
4953       ret->stream = ZINK_DGC_VBO;
4954       size = sizeof(VkBindVertexBufferIndirectCommandNV);
4955       break;
4956    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_INDEX_BUFFER_NV:
4957       ret->stream = ZINK_DGC_IB;
4958       size = sizeof(VkBindIndexBufferIndirectCommandNV);
4959       break;
4960    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_SHADER_GROUP_NV:
4961       ret->stream = ZINK_DGC_PSO;
4962       size = sizeof(VkBindShaderGroupIndirectCommandNV);
4963       break;
4964    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_PUSH_CONSTANT_NV:
4965       ret->stream = ZINK_DGC_PUSH;
4966       ret->pushconstantPipelineLayout = ctx->dgc.last_prog->base.layout;
4967       ret->pushconstantShaderStageFlags = VK_SHADER_STAGE_ALL_GRAPHICS;
4968       size = sizeof(float) * 6; //size for full tess level upload every time
4969       break;
4970    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_DRAW_NV:
4971       ret->stream = ZINK_DGC_DRAW;
4972       size = sizeof(VkDrawIndirectCommand);
4973       break;
4974    case VK_INDIRECT_COMMANDS_TOKEN_TYPE_DRAW_INDEXED_NV:
4975       ret->stream = ZINK_DGC_DRAW;
4976       size = sizeof(VkDrawIndexedIndirectCommand);
4977       break;
4978    default:
4979       unreachable("ack");
4980    }
4981    struct zink_resource *old = NULL;
4982    unsigned stream_count = screen->info.nv_dgc_props.maxIndirectCommandsStreamCount >= ZINK_DGC_MAX ? ZINK_DGC_MAX : 1;
4983    if (stream_count == 1)
4984       ret->stream = 0;
4985    unsigned stream = ret->stream;
4986    bool max_exceeded = !ctx->dgc.max_size[stream];
4987    ret->offset = ctx->dgc.cur_offsets[stream];
4988    if (ctx->dgc.buffers[stream]) {
4989       /* detect end of buffer */
4990       if (ctx->dgc.bind_offsets[stream] + ctx->dgc.cur_offsets[stream] + size > ctx->dgc.buffers[stream]->base.b.width0) {
4991          old = ctx->dgc.buffers[stream];
4992          ctx->dgc.buffers[stream] = NULL;
4993          max_exceeded = true;
4994       }
4995    }
4996    if (!ctx->dgc.buffers[stream]) {
4997       if (max_exceeded)
4998          ctx->dgc.max_size[stream] += size * 5;
4999       uint8_t *ptr;
5000       unsigned offset;
5001       u_upload_alloc(ctx->dgc.upload[stream], 0, ctx->dgc.max_size[stream],
5002                      screen->info.props.limits.minMemoryMapAlignment, &offset,
5003                      (struct pipe_resource **)&ctx->dgc.buffers[stream], (void **)&ptr);
5004       size_t cur_size = old ? (ctx->dgc.cur_offsets[stream] - ctx->dgc.bind_offsets[stream]) : 0;
5005       if (old) {
5006          struct pipe_resource *pold = &old->base.b;
5007          /* copy and delete old buffer */
5008          zink_batch_reference_resource_rw(&ctx->batch, old, true);
5009          memcpy(ptr + offset, ctx->dgc.maps[stream] + ctx->dgc.bind_offsets[stream], cur_size);
5010          pipe_resource_reference(&pold, NULL);
5011       }
5012       ctx->dgc.maps[stream] = ptr;
5013       ctx->dgc.bind_offsets[stream] = offset;
5014       ctx->dgc.cur_offsets[stream] = cur_size;
5015    }
5016    *mem = ctx->dgc.maps[stream] + ctx->dgc.cur_offsets[stream];
5017    ctx->dgc.cur_offsets[stream] += size;
5018    return ret;
5019 }
5020
5021 void
5022 zink_flush_dgc(struct zink_context *ctx)
5023 {
5024    struct zink_screen *screen = zink_screen(ctx->base.screen);
5025    struct zink_batch_state *bs = ctx->batch.state;
5026    if (!ctx->dgc.valid)
5027       return;
5028
5029    /* tokens should be created as they are used */
5030    unsigned num_cmds = util_dynarray_num_elements(&ctx->dgc.tokens, VkIndirectCommandsLayoutTokenNV);
5031    assert(num_cmds);
5032    VkIndirectCommandsLayoutTokenNV *cmds = ctx->dgc.tokens.data;
5033    uint32_t strides[ZINK_DGC_MAX] = {0};
5034
5035    unsigned stream_count = screen->info.nv_dgc_props.maxIndirectCommandsStreamCount >= ZINK_DGC_MAX ? ZINK_DGC_MAX : 1;
5036    VkIndirectCommandsStreamNV streams[ZINK_DGC_MAX];
5037    for (unsigned i = 0; i < stream_count; i++) {
5038       if (ctx->dgc.buffers[i]) {
5039          streams[i].buffer = ctx->dgc.buffers[i]->obj->buffer;
5040          streams[i].offset = ctx->dgc.bind_offsets[i];
5041       } else {
5042          streams[i].buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
5043          streams[i].offset = 0;
5044       }
5045    }
5046    /* this is a stupid pipeline that will never actually be used as anything but a container */
5047    VkPipeline pipeline = VK_NULL_HANDLE;
5048    if (screen->info.nv_dgc_props.maxGraphicsShaderGroupCount == 1) {
5049       /* RADV doesn't support shader pipeline binds, so use this hacky path */
5050       pipeline = ctx->gfx_pipeline_state.pipeline;
5051    } else {
5052       VkPrimitiveTopology vkmode = zink_primitive_topology(ctx->gfx_pipeline_state.gfx_prim_mode);
5053       pipeline = zink_create_gfx_pipeline(screen, ctx->dgc.last_prog, ctx->dgc.last_prog->objs, &ctx->gfx_pipeline_state, ctx->gfx_pipeline_state.element_state->binding_map, vkmode, false, &ctx->dgc.pipelines);
5054       assert(pipeline);
5055       util_dynarray_append(&bs->dgc.pipelines, VkPipeline, pipeline);
5056       VKCTX(CmdBindPipelineShaderGroupNV)(bs->cmdbuf, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline, 0);
5057    }
5058    unsigned remaining = num_cmds;
5059    for (unsigned i = 0; i < num_cmds; i += screen->info.nv_dgc_props.maxIndirectCommandsTokenCount, remaining -= screen->info.nv_dgc_props.maxIndirectCommandsTokenCount) {
5060       VkIndirectCommandsLayoutCreateInfoNV lci = {
5061          VK_STRUCTURE_TYPE_INDIRECT_COMMANDS_LAYOUT_CREATE_INFO_NV,
5062          NULL,
5063          0,
5064          VK_PIPELINE_BIND_POINT_GRAPHICS,
5065          MIN2(remaining, screen->info.nv_dgc_props.maxIndirectCommandsTokenCount),
5066          cmds + i,
5067          stream_count,
5068          strides
5069       };
5070       VkIndirectCommandsLayoutNV iclayout;
5071       VkResult res = VKSCR(CreateIndirectCommandsLayoutNV)(screen->dev, &lci, NULL, &iclayout);
5072       assert(res == VK_SUCCESS);
5073       util_dynarray_append(&bs->dgc.layouts, VkIndirectCommandsLayoutNV, iclayout);
5074
5075       /* a lot of hacks to set up a preprocess buffer */
5076       VkGeneratedCommandsMemoryRequirementsInfoNV info = {
5077          VK_STRUCTURE_TYPE_GENERATED_COMMANDS_MEMORY_REQUIREMENTS_INFO_NV,
5078          NULL,
5079          VK_PIPELINE_BIND_POINT_GRAPHICS,
5080          pipeline,
5081          iclayout,
5082          1
5083       };
5084       VkMemoryRequirements2 reqs = {
5085          VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2
5086       };
5087       VKSCR(GetGeneratedCommandsMemoryRequirementsNV)(screen->dev, &info, &reqs);
5088       struct pipe_resource templ = {0};
5089       templ.target = PIPE_BUFFER;
5090       templ.format = PIPE_FORMAT_R8_UNORM;
5091       templ.bind = 0;
5092       templ.usage = PIPE_USAGE_IMMUTABLE;
5093       templ.flags = 0;
5094       templ.width0 = reqs.memoryRequirements.size;
5095       templ.height0 = 1;
5096       templ.depth0 = 1;
5097       templ.array_size = 1;
5098       uint64_t params[] = {reqs.memoryRequirements.size, reqs.memoryRequirements.alignment, reqs.memoryRequirements.memoryTypeBits};
5099       struct pipe_resource *pres = screen->base.resource_create_with_modifiers(&screen->base, &templ, params, 3);
5100       assert(pres);
5101       zink_batch_reference_resource_rw(&ctx->batch, zink_resource(pres), true);
5102
5103       VkGeneratedCommandsInfoNV gen = {
5104          VK_STRUCTURE_TYPE_GENERATED_COMMANDS_INFO_NV,
5105          NULL,
5106          VK_PIPELINE_BIND_POINT_GRAPHICS,
5107          pipeline,
5108          iclayout,
5109          stream_count,
5110          streams,
5111          1,
5112          zink_resource(pres)->obj->buffer,
5113          0,
5114          pres->width0,
5115          VK_NULL_HANDLE,
5116          0,
5117          VK_NULL_HANDLE,
5118          0
5119       };
5120       VKCTX(CmdExecuteGeneratedCommandsNV)(ctx->batch.state->cmdbuf, VK_FALSE, &gen);
5121
5122       pipe_resource_reference(&pres, NULL);
5123    }
5124    util_dynarray_clear(&ctx->dgc.pipelines);
5125    util_dynarray_clear(&ctx->dgc.tokens);
5126    ctx->dgc.valid = false;
5127    ctx->pipeline_changed[0] = true;
5128    zink_select_draw_vbo(ctx);
5129 }
5130
5131 struct pipe_surface *
5132 zink_get_dummy_pipe_surface(struct zink_context *ctx, int samples_index)
5133 {
5134    if (!ctx->dummy_surface[samples_index]) {
5135       unsigned size = calc_max_dummy_fbo_size(ctx);
5136       ctx->dummy_surface[samples_index] = zink_surface_create_null(ctx, PIPE_TEXTURE_2D, size, size, BITFIELD_BIT(samples_index));
5137       /* This is possibly used with imageLoad which according to GL spec must return 0 */
5138       if (!samples_index) {
5139          union pipe_color_union color = {0};
5140          struct pipe_box box;
5141          u_box_2d(0, 0, size, size, &box);
5142          ctx->base.clear_texture(&ctx->base, ctx->dummy_surface[samples_index]->texture, 0, &box, &color);
5143       }
5144    }
5145    return ctx->dummy_surface[samples_index];
5146 }
5147
5148 struct zink_surface *
5149 zink_get_dummy_surface(struct zink_context *ctx, int samples_index)
5150 {
5151    return zink_csurface(zink_get_dummy_pipe_surface(ctx, samples_index));
5152
5153 }
5154
5155 static void
5156 zink_tc_parse_dsa(void *state, struct tc_renderpass_info *info)
5157 {
5158    struct zink_depth_stencil_alpha_state *cso = state;
5159    info->zsbuf_write_dsa |= (cso->hw_state.depth_write || cso->hw_state.stencil_test);
5160    info->zsbuf_read_dsa |= (cso->hw_state.depth_test || cso->hw_state.stencil_test);
5161    /* TODO: if zsbuf fbfetch is ever supported */
5162 }
5163
5164 static void
5165 zink_tc_parse_fs(void *state, struct tc_renderpass_info *info)
5166 {
5167    struct zink_shader *zs = state;
5168    info->zsbuf_write_fs |= zs->info.outputs_written & (BITFIELD64_BIT(FRAG_RESULT_DEPTH) | BITFIELD64_BIT(FRAG_RESULT_STENCIL));
5169    /* TODO: if >1 fbfetch attachment is ever supported */
5170    info->cbuf_fbfetch |= zs->info.fs.uses_fbfetch_output ? BITFIELD_BIT(0) : 0;
5171 }
5172
5173 void
5174 zink_parse_tc_info(struct zink_context *ctx)
5175 {
5176    struct tc_renderpass_info *info = &ctx->dynamic_fb.tc_info;
5177    /* reset cso info first */
5178    info->data16[2] = 0;
5179    if (ctx->gfx_stages[MESA_SHADER_FRAGMENT])
5180       zink_tc_parse_fs(ctx->gfx_stages[MESA_SHADER_FRAGMENT], info);
5181    if (ctx->dsa_state)
5182       zink_tc_parse_dsa(ctx->dsa_state, info);
5183    if (ctx->zsbuf_unused == zink_is_zsbuf_used(ctx))
5184       ctx->rp_layout_changed = true;
5185 }
5186
5187 struct pipe_context *
5188 zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
5189 {
5190    struct zink_screen *screen = zink_screen(pscreen);
5191    struct zink_context *ctx = rzalloc(NULL, struct zink_context);
5192    bool is_copy_only = (flags & ZINK_CONTEXT_COPY_ONLY) > 0;
5193    bool is_compute_only = (flags & PIPE_CONTEXT_COMPUTE_ONLY) > 0;
5194    bool is_robust = (flags & PIPE_CONTEXT_ROBUST_BUFFER_ACCESS) > 0;
5195    if (!ctx)
5196       goto fail;
5197
5198    ctx->flags = flags;
5199    ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
5200    ctx->gfx_pipeline_state.dirty = true;
5201    ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = 1;
5202    ctx->gfx_pipeline_state.uses_dynamic_stride = screen->info.have_EXT_extended_dynamic_state ||
5203                                                  screen->info.have_EXT_vertex_input_dynamic_state;
5204    ctx->compute_pipeline_state.dirty = true;
5205    ctx->fb_changed = ctx->rp_changed = true;
5206    ctx->sample_mask_changed = true;
5207    ctx->gfx_pipeline_state.gfx_prim_mode = MESA_PRIM_COUNT;
5208    ctx->gfx_pipeline_state.shader_rast_prim = MESA_PRIM_COUNT;
5209    ctx->gfx_pipeline_state.rast_prim = MESA_PRIM_COUNT;
5210
5211    zink_init_draw_functions(ctx, screen);
5212    zink_init_grid_functions(ctx);
5213
5214    ctx->base.screen = pscreen;
5215    ctx->base.priv = priv;
5216
5217    ctx->base.destroy = zink_context_destroy;
5218    ctx->base.set_debug_callback = zink_set_debug_callback;
5219    ctx->base.get_device_reset_status = zink_get_device_reset_status;
5220    ctx->base.set_device_reset_callback = zink_set_device_reset_callback;
5221
5222    zink_context_state_init(&ctx->base);
5223
5224    ctx->base.create_sampler_state = zink_create_sampler_state;
5225    ctx->base.bind_sampler_states = screen->info.have_EXT_non_seamless_cube_map ? zink_bind_sampler_states : zink_bind_sampler_states_nonseamless;
5226    ctx->base.delete_sampler_state = zink_delete_sampler_state;
5227
5228    ctx->base.create_sampler_view = zink_create_sampler_view;
5229    ctx->base.set_sampler_views = zink_set_sampler_views;
5230    ctx->base.sampler_view_destroy = zink_sampler_view_destroy;
5231    ctx->base.get_sample_position = zink_get_sample_position;
5232    ctx->base.set_sample_locations = zink_set_sample_locations;
5233
5234    zink_program_init(ctx);
5235
5236    ctx->base.set_polygon_stipple = zink_set_polygon_stipple;
5237    ctx->base.set_vertex_buffers = zink_set_vertex_buffers;
5238    ctx->base.set_viewport_states = zink_set_viewport_states;
5239    ctx->base.set_scissor_states = zink_set_scissor_states;
5240    ctx->base.set_inlinable_constants = zink_set_inlinable_constants;
5241    ctx->base.set_constant_buffer = zink_set_constant_buffer;
5242    ctx->base.set_shader_buffers = zink_set_shader_buffers;
5243    ctx->base.set_shader_images = zink_set_shader_images;
5244    ctx->base.set_framebuffer_state = zink_set_framebuffer_state;
5245    ctx->base.set_stencil_ref = zink_set_stencil_ref;
5246    ctx->base.set_clip_state = zink_set_clip_state;
5247    ctx->base.set_blend_color = zink_set_blend_color;
5248    ctx->base.set_tess_state = zink_set_tess_state;
5249    ctx->base.set_patch_vertices = zink_set_patch_vertices;
5250
5251    ctx->base.set_min_samples = zink_set_min_samples;
5252    ctx->gfx_pipeline_state.min_samples = 0;
5253    ctx->base.set_sample_mask = zink_set_sample_mask;
5254    ctx->gfx_pipeline_state.sample_mask = UINT32_MAX;
5255
5256    ctx->base.clear = zink_clear;
5257    ctx->base.clear_texture = screen->info.have_KHR_dynamic_rendering ? zink_clear_texture_dynamic : zink_clear_texture;
5258    ctx->base.clear_buffer = zink_clear_buffer;
5259    ctx->base.clear_render_target = zink_clear_render_target;
5260    ctx->base.clear_depth_stencil = zink_clear_depth_stencil;
5261
5262    ctx->base.create_fence_fd = zink_create_fence_fd;
5263    ctx->base.fence_server_sync = zink_fence_server_sync;
5264    ctx->base.fence_server_signal = zink_fence_server_signal;
5265    ctx->base.flush = zink_flush;
5266    ctx->base.memory_barrier = zink_memory_barrier;
5267    ctx->base.texture_barrier = zink_texture_barrier;
5268    ctx->base.evaluate_depth_buffer = zink_evaluate_depth_buffer;
5269
5270    ctx->base.resource_commit = zink_resource_commit;
5271    ctx->base.resource_copy_region = zink_resource_copy_region;
5272    ctx->base.blit = zink_blit;
5273    ctx->base.create_stream_output_target = zink_create_stream_output_target;
5274    ctx->base.stream_output_target_destroy = zink_stream_output_target_destroy;
5275
5276    ctx->base.set_stream_output_targets = zink_set_stream_output_targets;
5277    ctx->base.flush_resource = zink_flush_resource;
5278    if (screen->info.have_KHR_buffer_device_address)
5279       ctx->base.set_global_binding = zink_set_global_binding;
5280
5281    ctx->base.emit_string_marker = zink_emit_string_marker;
5282
5283    zink_context_surface_init(&ctx->base);
5284    zink_context_resource_init(&ctx->base);
5285    zink_context_query_init(&ctx->base);
5286
5287    list_inithead(&ctx->query_pools);
5288    _mesa_set_init(&ctx->update_barriers[0][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5289    _mesa_set_init(&ctx->update_barriers[1][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5290    _mesa_set_init(&ctx->update_barriers[0][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5291    _mesa_set_init(&ctx->update_barriers[1][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5292    ctx->need_barriers[0] = &ctx->update_barriers[0][0];
5293    ctx->need_barriers[1] = &ctx->update_barriers[1][0];
5294
5295    slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
5296    slab_create_child(&ctx->transfer_pool_unsync, &screen->transfer_pool);
5297
5298    ctx->base.stream_uploader = u_upload_create_default(&ctx->base);
5299    ctx->base.const_uploader = u_upload_create_default(&ctx->base);
5300    for (int i = 0; i < ARRAY_SIZE(ctx->fb_clears); i++)
5301       util_dynarray_init(&ctx->fb_clears[i].clears, ctx);
5302
5303    if (zink_debug & ZINK_DEBUG_DGC) {
5304       util_dynarray_init(&ctx->dgc.pipelines, ctx);
5305       util_dynarray_init(&ctx->dgc.tokens, ctx);
5306       for (unsigned i = 0; i < ARRAY_SIZE(ctx->dgc.upload); i++)
5307          ctx->dgc.upload[i] = u_upload_create_default(&ctx->base);
5308    }
5309
5310    if (!is_copy_only) {
5311       ctx->blitter = util_blitter_create(&ctx->base);
5312       if (!ctx->blitter)
5313          goto fail;
5314    }
5315
5316    zink_set_last_vertex_key(ctx)->last_vertex_stage = true;
5317    ctx->gfx_pipeline_state.shader_keys.last_vertex.key.vs_base.last_vertex_stage = true;
5318    zink_set_tcs_key_patches(ctx, 1);
5319    if (!screen->optimal_keys) {
5320       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_VERTEX].size = sizeof(struct zink_vs_key_base);
5321       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_EVAL].size = sizeof(struct zink_vs_key_base);
5322       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_TESS_CTRL].size = sizeof(struct zink_tcs_key);
5323       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_GEOMETRY].size = sizeof(struct zink_gs_key);
5324       ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].size = sizeof(struct zink_fs_key);
5325
5326       /* this condition must be updated if new fields are added to zink_cs_key */
5327       if (screen->driver_workarounds.lower_robustImageAccess2)
5328           ctx->compute_pipeline_state.key.size = sizeof(struct zink_cs_key);
5329
5330       if (is_robust && screen->driver_workarounds.lower_robustImageAccess2) {
5331          ctx->compute_pipeline_state.key.key.cs.robust_access = true;
5332          for (gl_shader_stage pstage = MESA_SHADER_VERTEX; pstage < MESA_SHADER_FRAGMENT; pstage++)
5333             ctx->gfx_pipeline_state.shader_keys.key[pstage].key.vs_base.robust_access = true;
5334          ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].key.fs.robust_access = true;
5335       }
5336    }
5337    _mesa_hash_table_init(&ctx->framebuffer_cache, ctx, hash_framebuffer_imageless, equals_framebuffer_imageless);
5338    if (!zink_init_render_pass(ctx))
5339       goto fail;
5340    for (unsigned i = 0; i < ARRAY_SIZE(ctx->rendering_state_cache); i++)
5341       _mesa_set_init(&ctx->rendering_state_cache[i], ctx, hash_rendering_state, equals_rendering_state);
5342    ctx->dynamic_fb.info.pColorAttachments = ctx->dynamic_fb.attachments;
5343    ctx->dynamic_fb.info.sType = VK_STRUCTURE_TYPE_RENDERING_INFO;
5344    for (unsigned i = 0; i < ARRAY_SIZE(ctx->dynamic_fb.attachments); i++) {
5345       VkRenderingAttachmentInfo *att = &ctx->dynamic_fb.attachments[i];
5346       att->sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO;
5347       att->imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
5348       att->storeOp = VK_ATTACHMENT_STORE_OP_STORE;
5349    }
5350    ctx->gfx_pipeline_state.rendering_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO;
5351    ctx->gfx_pipeline_state.rendering_info.pColorAttachmentFormats = ctx->gfx_pipeline_state.rendering_formats;
5352    ctx->gfx_pipeline_state.feedback_loop = screen->driver_workarounds.always_feedback_loop;
5353    ctx->gfx_pipeline_state.feedback_loop_zs = screen->driver_workarounds.always_feedback_loop_zs;
5354
5355    const uint32_t data[] = {0};
5356    if (!is_copy_only) {
5357       ctx->dummy_vertex_buffer = pipe_buffer_create(&screen->base,
5358          PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_SHADER_IMAGE, PIPE_USAGE_IMMUTABLE, sizeof(data));
5359       if (!ctx->dummy_vertex_buffer)
5360          goto fail;
5361       ctx->dummy_xfb_buffer = pipe_buffer_create(&screen->base,
5362          PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_IMMUTABLE, sizeof(data));
5363       if (!ctx->dummy_xfb_buffer)
5364          goto fail;
5365    }
5366    if (!is_copy_only) {
5367       VkBufferViewCreateInfo bvci = create_bvci(ctx, zink_resource(ctx->dummy_vertex_buffer), PIPE_FORMAT_R8G8B8A8_UNORM, 0, sizeof(data));
5368       ctx->dummy_bufferview = get_buffer_view(ctx, zink_resource(ctx->dummy_vertex_buffer), &bvci);
5369       if (!ctx->dummy_bufferview)
5370          goto fail;
5371
5372       if (!zink_descriptors_init(ctx))
5373          goto fail;
5374    }
5375
5376    if (!is_copy_only && !is_compute_only) {
5377       ctx->base.create_texture_handle = zink_create_texture_handle;
5378       ctx->base.delete_texture_handle = zink_delete_texture_handle;
5379       ctx->base.make_texture_handle_resident = zink_make_texture_handle_resident;
5380       ctx->base.create_image_handle = zink_create_image_handle;
5381       ctx->base.delete_image_handle = zink_delete_image_handle;
5382       ctx->base.make_image_handle_resident = zink_make_image_handle_resident;
5383       for (unsigned i = 0; i < 2; i++) {
5384          _mesa_hash_table_init(&ctx->di.bindless[i].img_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5385          _mesa_hash_table_init(&ctx->di.bindless[i].tex_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
5386
5387          /* allocate 1024 slots and reserve slot 0 */
5388          util_idalloc_init(&ctx->di.bindless[i].tex_slots, ZINK_MAX_BINDLESS_HANDLES);
5389          util_idalloc_alloc(&ctx->di.bindless[i].tex_slots);
5390          util_idalloc_init(&ctx->di.bindless[i].img_slots, ZINK_MAX_BINDLESS_HANDLES);
5391          util_idalloc_alloc(&ctx->di.bindless[i].img_slots);
5392          if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5393             ctx->di.bindless[i].db.buffer_infos = malloc(sizeof(VkDescriptorAddressInfoEXT) * ZINK_MAX_BINDLESS_HANDLES);
5394             if (!ctx->di.bindless[i].db.buffer_infos) {
5395                mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].db.buffer_infos!",i);
5396                goto fail;
5397             }
5398             for (unsigned j = 0; j < ZINK_MAX_BINDLESS_HANDLES; j++) {
5399                ctx->di.bindless[i].db.buffer_infos[j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5400                ctx->di.bindless[i].db.buffer_infos[j].pNext = NULL;
5401             }
5402          } else {
5403             ctx->di.bindless[i].t.buffer_infos = malloc(sizeof(VkBufferView) * ZINK_MAX_BINDLESS_HANDLES);
5404             if (!ctx->di.bindless[i].t.buffer_infos) {
5405                mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].t.buffer_infos!",i);
5406                goto fail;
5407             }
5408          }
5409          ctx->di.bindless[i].img_infos = malloc(sizeof(VkDescriptorImageInfo) * ZINK_MAX_BINDLESS_HANDLES);
5410          if (!ctx->di.bindless[i].img_infos) {
5411             mesa_loge("ZINK: failed to allocate ctx->di.bindless[%d].img_infos!",i);
5412             goto fail;
5413          }
5414          util_dynarray_init(&ctx->di.bindless[i].updates, NULL);
5415          util_dynarray_init(&ctx->di.bindless[i].resident, NULL);
5416       }
5417    }
5418
5419    zink_start_batch(ctx, &ctx->batch);
5420    if (!ctx->batch.state)
5421       goto fail;
5422
5423    if (screen->compact_descriptors)
5424       ctx->invalidate_descriptor_state = zink_context_invalidate_descriptor_state_compact;
5425    else
5426       ctx->invalidate_descriptor_state = zink_context_invalidate_descriptor_state;
5427    if (!is_copy_only && !is_compute_only) {
5428       pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_vertex_buffer, 0, sizeof(data), data);
5429       pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_xfb_buffer, 0, sizeof(data), data);
5430       reapply_color_write(ctx);
5431
5432       /* set on startup just to avoid validation errors if a draw comes through without
5433       * a tess shader later
5434       */
5435       if (screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) {
5436          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, 1);
5437          VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->barrier_cmdbuf, 1);
5438       }
5439    }
5440    if (!is_copy_only) {
5441       for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
5442          /* need to update these based on screen config for null descriptors */
5443          for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ubos[i]); j++) {
5444             update_descriptor_state_ubo(ctx, i, j, NULL);
5445             if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5446                ctx->di.db.ubos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5447          }
5448          for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.textures[i]); j++) {
5449             update_descriptor_state_sampler(ctx, i, j, NULL);
5450             if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5451                ctx->di.db.tbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5452          }
5453          for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ssbos[i]); j++) {
5454             update_descriptor_state_ssbo(ctx, i, j, NULL);
5455             if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5456                ctx->di.db.ssbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5457          }
5458          for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.images[i]); j++) {
5459             update_descriptor_state_image(ctx, i, j, NULL);
5460             if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
5461                ctx->di.db.texel_images[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT;
5462          }
5463       }
5464       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) {
5465          /* cache null fbfetch descriptor info */
5466          ctx->di.fbfetch.imageView = zink_get_dummy_surface(ctx, 0)->image_view;
5467          ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
5468          VkDescriptorGetInfoEXT info;
5469          info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT;
5470          info.pNext = NULL;
5471          info.type = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT;
5472          info.data.pInputAttachmentImage = &ctx->di.fbfetch;
5473          if (screen->info.db_props.inputAttachmentDescriptorSize)
5474             VKSCR(GetDescriptorEXT)(screen->dev, &info, screen->info.db_props.inputAttachmentDescriptorSize, ctx->di.fbfetch_db);
5475          memset(&ctx->di.fbfetch, 0, sizeof(ctx->di.fbfetch));
5476       }
5477       if (!screen->info.rb2_feats.nullDescriptor)
5478          ctx->di.fbfetch.imageView = zink_get_dummy_surface(ctx, 0)->image_view;
5479
5480       p_atomic_inc(&screen->base.num_contexts);
5481    }
5482
5483    zink_select_draw_vbo(ctx);
5484    zink_select_launch_grid(ctx);
5485
5486    if (!is_copy_only && zink_debug & ZINK_DEBUG_SHADERDB) {
5487       if (!screen->info.have_EXT_vertex_input_dynamic_state) {
5488          struct pipe_vertex_element velems[32] = {0};
5489          for (unsigned i = 0; i < ARRAY_SIZE(velems); i++)
5490             velems[i].src_format = PIPE_FORMAT_R8G8B8_UNORM;
5491          void *state = ctx->base.create_vertex_elements_state(&ctx->base, ARRAY_SIZE(velems), velems);
5492          ctx->base.bind_vertex_elements_state(&ctx->base, state);
5493       }
5494       ctx->gfx_pipeline_state.sample_mask = BITFIELD_MASK(32);
5495       struct pipe_framebuffer_state fb = {0};
5496       fb.cbufs[0] = zink_get_dummy_pipe_surface(ctx, 0);
5497       fb.nr_cbufs = 1;
5498       fb.width = fb.height = 256;
5499       ctx->base.set_framebuffer_state(&ctx->base, &fb);
5500       ctx->disable_fs = true;
5501       struct pipe_depth_stencil_alpha_state dsa = {0};
5502       void *state = ctx->base.create_depth_stencil_alpha_state(&ctx->base, &dsa);
5503       ctx->base.bind_depth_stencil_alpha_state(&ctx->base, state);
5504
5505       struct pipe_blend_state blend = {
5506          .rt[0].colormask = 0xF
5507       };
5508
5509       void *blend_state = ctx->base.create_blend_state(&ctx->base, &blend);
5510       ctx->base.bind_blend_state(&ctx->base, blend_state);
5511
5512       zink_batch_rp(ctx);
5513    }
5514
5515    if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) {
5516       return &ctx->base;
5517    }
5518
5519    struct threaded_context *tc = (struct threaded_context*)threaded_context_create(&ctx->base, &screen->transfer_pool,
5520                                                      zink_context_replace_buffer_storage,
5521                                                      &(struct threaded_context_options){
5522                                                         .create_fence = zink_create_tc_fence_for_tc,
5523                                                         .is_resource_busy = zink_context_is_resource_busy,
5524                                                         .driver_calls_flush_notify = !screen->driver_workarounds.track_renderpasses,
5525                                                         .unsynchronized_get_device_reset_status = true,
5526                                                         .parse_renderpass_info = screen->driver_workarounds.track_renderpasses,
5527                                                         .dsa_parse = zink_tc_parse_dsa,
5528                                                         .fs_parse = zink_tc_parse_fs,
5529                                                      },
5530                                                      &ctx->tc);
5531
5532    if (tc && (struct zink_context*)tc != ctx) {
5533       ctx->track_renderpasses = screen->driver_workarounds.track_renderpasses;
5534       threaded_context_init_bytes_mapped_limit(tc, 4);
5535       ctx->base.set_context_param = zink_set_context_param;
5536    }
5537
5538    return (struct pipe_context*)tc;
5539
5540 fail:
5541    if (ctx)
5542       zink_context_destroy(&ctx->base);
5543    return NULL;
5544 }
5545
5546 struct zink_context *
5547 zink_tc_context_unwrap(struct pipe_context *pctx, bool threaded)
5548 {
5549    /* need to get the actual zink_context, not the threaded context */
5550    if (threaded)
5551       pctx = threaded_context_unwrap_sync(pctx);
5552    pctx = trace_get_possibly_threaded_context(pctx);
5553    return zink_context(pctx);
5554 }
5555
5556
5557 static bool
5558 add_implicit_feedback_loop(struct zink_context *ctx, struct zink_resource *res)
5559 {
5560    /* can only feedback loop with fb+sampler bind; image bind must be GENERAL */
5561    if (!res->fb_bind_count || !res->sampler_bind_count[0] || res->image_bind_count[0])
5562       return false;
5563    if (!(res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) && !zink_is_zsbuf_write(ctx))
5564       /* if zsbuf isn't used then it effectively has no fb binds */
5565       /* if zsbuf isn't written to then it'll be fine with read-only access */
5566       return false;
5567    bool is_feedback = false;
5568    /* avoid false positives when a texture is bound but not used */
5569    u_foreach_bit(vkstage, res->gfx_barrier) {
5570       VkPipelineStageFlags vkstagebit = BITFIELD_BIT(vkstage);
5571       if (vkstagebit < VK_PIPELINE_STAGE_VERTEX_SHADER_BIT || vkstagebit > VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT)
5572          continue;
5573       /* in-range VkPipelineStageFlagBits can be converted to VkShaderStageFlags with a bitshift */
5574       gl_shader_stage stage = vk_to_mesa_shader_stage((VkShaderStageFlagBits)(vkstagebit >> 3));
5575       /* check shader texture usage against resource's sampler binds */
5576       if ((ctx->gfx_stages[stage] && (res->sampler_binds[stage] & ctx->gfx_stages[stage]->info.textures_used[0])))
5577          is_feedback = true;
5578    }
5579    if (!is_feedback)
5580       return false;
5581    if (ctx->feedback_loops & res->fb_binds)
5582       /* already added */
5583       return true;
5584    /* new feedback loop detected */
5585    if (res->aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
5586       if (!ctx->gfx_pipeline_state.feedback_loop)
5587          ctx->gfx_pipeline_state.dirty = true;
5588       ctx->gfx_pipeline_state.feedback_loop = true;
5589    } else {
5590       if (!ctx->gfx_pipeline_state.feedback_loop_zs)
5591          ctx->gfx_pipeline_state.dirty = true;
5592       ctx->gfx_pipeline_state.feedback_loop_zs = true;
5593    }
5594    ctx->rp_layout_changed = true;
5595    ctx->feedback_loops |= res->fb_binds;
5596    u_foreach_bit(idx, res->fb_binds) {
5597       if (zink_screen(ctx->base.screen)->info.have_EXT_attachment_feedback_loop_layout)
5598          ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT;
5599       else
5600          ctx->dynamic_fb.attachments[idx].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
5601    }
5602    update_feedback_loop_dynamic_state(ctx);
5603    return true;
5604 }
5605
5606 void
5607 zink_update_barriers(struct zink_context *ctx, bool is_compute,
5608                      struct pipe_resource *index, struct pipe_resource *indirect, struct pipe_resource *indirect_draw_count)
5609 {
5610    assert(!ctx->blitting);
5611    if (!ctx->need_barriers[is_compute]->entries)
5612       return;
5613    struct set *need_barriers = ctx->need_barriers[is_compute];
5614    ctx->barrier_set_idx[is_compute] = !ctx->barrier_set_idx[is_compute];
5615    ctx->need_barriers[is_compute] = &ctx->update_barriers[is_compute][ctx->barrier_set_idx[is_compute]];
5616    ASSERTED bool check_rp = ctx->batch.in_rp && ctx->dynamic_fb.tc_info.zsbuf_invalidate;
5617    set_foreach(need_barriers, he) {
5618       struct zink_resource *res = (struct zink_resource *)he->key;
5619       if (res->bind_count[is_compute]) {
5620          VkPipelineStageFlagBits pipeline = is_compute ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : res->gfx_barrier;
5621          if (res->base.b.target == PIPE_BUFFER)
5622             zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, res->barrier_access[is_compute], pipeline);
5623          else {
5624             bool is_feedback = is_compute ? false : add_implicit_feedback_loop(ctx, res);
5625             VkImageLayout layout = zink_descriptor_util_image_layout_eval(ctx, res, is_compute);
5626             /* GENERAL is only used for feedback loops and storage image binds */
5627             if (is_feedback || layout != VK_IMAGE_LAYOUT_GENERAL || res->image_bind_count[is_compute])
5628                zink_screen(ctx->base.screen)->image_barrier(ctx, res, layout, res->barrier_access[is_compute], pipeline);
5629             assert(!check_rp || check_rp == ctx->batch.in_rp);
5630             if (is_feedback)
5631                update_res_sampler_layouts(ctx, res);
5632          }
5633          if (zink_resource_access_is_write(res->barrier_access[is_compute]) ||
5634              // TODO: figure out a way to link up layouts between unordered and main cmdbuf
5635              res->base.b.target != PIPE_BUFFER)
5636             res->obj->unordered_write = false;
5637          res->obj->unordered_read = false;
5638          /* always barrier on draw if this resource has either multiple image write binds or
5639           * image write binds and image read binds
5640           */
5641          if (res->write_bind_count[is_compute] && res->bind_count[is_compute] > 1)
5642             _mesa_set_add_pre_hashed(ctx->need_barriers[is_compute], he->hash, res);
5643       }
5644       _mesa_set_remove(need_barriers, he);
5645       if (!need_barriers->entries)
5646          break;
5647    }
5648 }
5649
5650 /**
5651  * Emits a debug marker in the cmd stream to be captured by perfetto during
5652  * execution on the GPU.
5653  */
5654 bool
5655 zink_cmd_debug_marker_begin(struct zink_context *ctx, VkCommandBuffer cmdbuf, const char *fmt, ...)
5656 {
5657    if (!zink_tracing)
5658       return false;
5659
5660    char *name;
5661    va_list va;
5662    va_start(va, fmt);
5663    int ret = vasprintf(&name, fmt, va);
5664    va_end(va);
5665
5666    if (ret == -1)
5667       return false;
5668
5669    VkDebugUtilsLabelEXT info = { 0 };
5670    info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT;
5671    info.pLabelName = name;
5672
5673    VKCTX(CmdBeginDebugUtilsLabelEXT)(cmdbuf ? cmdbuf : ctx->batch.state->cmdbuf, &info);
5674
5675    free(name);
5676    return true;
5677 }
5678
5679 void
5680 zink_cmd_debug_marker_end(struct zink_context *ctx, VkCommandBuffer cmdbuf, bool emitted)
5681 {
5682    if (emitted)
5683       VKCTX(CmdEndDebugUtilsLabelEXT)(cmdbuf);
5684 }