src/gallium/drivers/zink/zink_resource.c

   1 /*
   2  * Copyright 2018 Collabora Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "zink_resource.h"
  25
  26 #include "zink_batch.h"
  27 #include "zink_clear.h"
  28 #include "zink_context.h"
  29 #include "zink_fence.h"
  30 #include "zink_program.h"
  31 #include "zink_screen.h"
  32 #include "zink_kopper.h"
  33
  34 #ifdef VK_USE_PLATFORM_METAL_EXT
  35 #include "QuartzCore/CAMetalLayer.h"
  36 #endif
  37
  38 #include "vk_format.h"
  39 #include "util/u_blitter.h"
  40 #include "util/u_debug.h"
  41 #include "util/format/u_format.h"
  42 #include "util/u_transfer_helper.h"
  43 #include "util/u_inlines.h"
  44 #include "util/u_memory.h"
  45 #include "util/u_upload_mgr.h"
  46 #include "util/os_file.h"
  47 #include "frontend/winsys_handle.h"
  48
  49 #if !defined(__APPLE__)
  50 #define ZINK_USE_DMABUF
  51 #endif
  52
  53 #if defined(ZINK_USE_DMABUF) && !defined(_WIN32)
  54 #include "drm-uapi/drm_fourcc.h"
  55 #else
  56 /* these won't actually be used */
  57 #define DRM_FORMAT_MOD_INVALID 0
  58 #define DRM_FORMAT_MOD_LINEAR 0
  59 #endif
  60
  61 #if defined(__APPLE__)
  62 // Source of MVK_VERSION
  63 #include "MoltenVK/vk_mvk_moltenvk.h"
  64 #endif
  65
  66 #define ZINK_EXTERNAL_MEMORY_HANDLE 999
  67
  68 static bool
  69 equals_ivci(const void *a, const void *b)
  70 {
  71    const uint8_t *pa = a;
  72    const uint8_t *pb = b;
  73    size_t offset = offsetof(VkImageViewCreateInfo, flags);
  74    return memcmp(pa + offset, pb + offset, sizeof(VkImageViewCreateInfo) - offset) == 0;
  75 }
  76
  77 static bool
  78 equals_bvci(const void *a, const void *b)
  79 {
  80    const uint8_t *pa = a;
  81    const uint8_t *pb = b;
  82    size_t offset = offsetof(VkBufferViewCreateInfo, flags);
  83    return memcmp(pa + offset, pb + offset, sizeof(VkBufferViewCreateInfo) - offset) == 0;
  84 }
  85
  86 static void
  87 zink_transfer_flush_region(struct pipe_context *pctx,
  88                            struct pipe_transfer *ptrans,
  89                            const struct pipe_box *box);
  90
  91 void
  92 debug_describe_zink_resource_object(char *buf, const struct zink_resource_object *ptr)
  93 {
  94    sprintf(buf, "zink_resource_object");
  95 }
  96
  97 void
  98 zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_object *obj)
  99 {
 100    if (obj->is_buffer) {
 101       while (util_dynarray_contains(&obj->views, VkBufferView))
 102          VKSCR(DestroyBufferView)(screen->dev, util_dynarray_pop(&obj->views, VkBufferView), NULL);
 103    } else {
 104       while (util_dynarray_contains(&obj->views, VkImageView))
 105          VKSCR(DestroyImageView)(screen->dev, util_dynarray_pop(&obj->views, VkImageView), NULL);
 106    }
 107    util_dynarray_fini(&obj->views);
 108    for (unsigned i = 0; i < ARRAY_SIZE(obj->copies); i++)
 109       util_dynarray_fini(&obj->copies[i]);
 110    if (obj->is_buffer) {
 111       VKSCR(DestroyBuffer)(screen->dev, obj->buffer, NULL);
 112       VKSCR(DestroyBuffer)(screen->dev, obj->storage_buffer, NULL);
 113    } else if (obj->dt) {
 114       zink_kopper_displaytarget_destroy(screen, obj->dt);
 115    } else if (!obj->is_aux) {
 116       VKSCR(DestroyImage)(screen->dev, obj->image, NULL);
 117    } else {
 118 #if defined(ZINK_USE_DMABUF) && !defined(_WIN32)
 119       close(obj->handle);
 120 #endif
 121    }
 122
 123    simple_mtx_destroy(&obj->view_lock);
 124    if (obj->dt) {
 125       FREE(obj->bo); //this is a dummy struct
 126    } else
 127       zink_bo_unref(screen, obj->bo);
 128    FREE(obj);
 129 }
 130
 131 static void
 132 zink_resource_destroy(struct pipe_screen *pscreen,
 133                       struct pipe_resource *pres)
 134 {
 135    struct zink_screen *screen = zink_screen(pscreen);
 136    struct zink_resource *res = zink_resource(pres);
 137    if (pres->target == PIPE_BUFFER) {
 138       util_range_destroy(&res->valid_buffer_range);
 139       util_idalloc_mt_free(&screen->buffer_ids, res->base.buffer_id_unique);
 140       assert(!_mesa_hash_table_num_entries(&res->bufferview_cache));
 141       simple_mtx_destroy(&res->bufferview_mtx);
 142       ralloc_free(res->bufferview_cache.table);
 143    } else {
 144       assert(!_mesa_hash_table_num_entries(&res->surface_cache));
 145       simple_mtx_destroy(&res->surface_mtx);
 146       ralloc_free(res->surface_cache.table);
 147    }
 148    /* no need to do anything for the caches, these objects own the resource lifetimes */
 149
 150    zink_resource_object_reference(screen, &res->obj, NULL);
 151    threaded_resource_deinit(pres);
 152    FREE_CL(res);
 153 }
 154
 155 static VkImageAspectFlags
 156 aspect_from_format(enum pipe_format fmt)
 157 {
 158    if (util_format_is_depth_or_stencil(fmt)) {
 159       VkImageAspectFlags aspect = 0;
 160       const struct util_format_description *desc = util_format_description(fmt);
 161       if (util_format_has_depth(desc))
 162          aspect |= VK_IMAGE_ASPECT_DEPTH_BIT;
 163       if (util_format_has_stencil(desc))
 164          aspect |= VK_IMAGE_ASPECT_STENCIL_BIT;
 165       return aspect;
 166    } else
 167      return VK_IMAGE_ASPECT_COLOR_BIT;
 168 }
 169
 170 static VkBufferCreateInfo
 171 create_bci(struct zink_screen *screen, const struct pipe_resource *templ, unsigned bind)
 172 {
 173    VkBufferCreateInfo bci;
 174    bci.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
 175    bci.pNext = NULL;
 176    bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
 177    bci.queueFamilyIndexCount = 0;
 178    bci.pQueueFamilyIndices = NULL;
 179    bci.size = templ->width0;
 180    bci.flags = 0;
 181    assert(bci.size > 0);
 182
 183    if (bind & ZINK_BIND_DESCRIPTOR) {
 184       /* gallium sizes are all uint32_t, while the total size of this buffer may exceed that limit */
 185       bci.usage = 0;
 186       if (bind & ZINK_BIND_SAMPLER_DESCRIPTOR)
 187          bci.usage |= VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT;
 188       if (bind & ZINK_BIND_RESOURCE_DESCRIPTOR)
 189          bci.usage |= VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT;
 190    } else {
 191       bci.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
 192                   VK_BUFFER_USAGE_TRANSFER_DST_BIT |
 193                   VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
 194
 195       bci.usage |= VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT |
 196                   VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |
 197                   VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
 198                   VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
 199                   VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
 200                   VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT |
 201                   VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT;
 202    }
 203    if (screen->info.have_KHR_buffer_device_address)
 204       bci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
 205
 206    if (bind & PIPE_BIND_SHADER_IMAGE)
 207       bci.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
 208
 209    if (bind & PIPE_BIND_QUERY_BUFFER)
 210       bci.usage |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT;
 211
 212    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 213       bci.flags |= VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
 214    return bci;
 215 }
 216
 217 static bool
 218 check_ici(struct zink_screen *screen, VkImageCreateInfo *ici, uint64_t modifier)
 219 {
 220    VkImageFormatProperties image_props;
 221    VkResult ret;
 222    assert(modifier == DRM_FORMAT_MOD_INVALID ||
 223           (VKSCR(GetPhysicalDeviceImageFormatProperties2) && screen->info.have_EXT_image_drm_format_modifier));
 224    if (VKSCR(GetPhysicalDeviceImageFormatProperties2)) {
 225       VkImageFormatProperties2 props2;
 226       props2.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2;
 227       props2.pNext = NULL;
 228       VkSamplerYcbcrConversionImageFormatProperties ycbcr_props;
 229       ycbcr_props.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_IMAGE_FORMAT_PROPERTIES;
 230       ycbcr_props.pNext = NULL;
 231       if (screen->info.have_KHR_sampler_ycbcr_conversion)
 232          props2.pNext = &ycbcr_props;
 233       VkPhysicalDeviceImageFormatInfo2 info;
 234       info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2;
 235       /* possibly VkImageFormatListCreateInfo */
 236       info.pNext = ici->pNext;
 237       info.format = ici->format;
 238       info.type = ici->imageType;
 239       info.tiling = ici->tiling;
 240       info.usage = ici->usage;
 241       info.flags = ici->flags;
 242
 243       VkPhysicalDeviceImageDrmFormatModifierInfoEXT mod_info;
 244       if (modifier != DRM_FORMAT_MOD_INVALID) {
 245          mod_info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT;
 246          mod_info.pNext = info.pNext;
 247          mod_info.drmFormatModifier = modifier;
 248          mod_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
 249          mod_info.queueFamilyIndexCount = 0;
 250          info.pNext = &mod_info;
 251       }
 252
 253       ret = VKSCR(GetPhysicalDeviceImageFormatProperties2)(screen->pdev, &info, &props2);
 254       /* this is using VK_IMAGE_CREATE_EXTENDED_USAGE_BIT and can't be validated */
 255       if (vk_format_aspects(ici->format) & VK_IMAGE_ASPECT_PLANE_1_BIT)
 256          ret = VK_SUCCESS;
 257       image_props = props2.imageFormatProperties;
 258    } else
 259       ret = VKSCR(GetPhysicalDeviceImageFormatProperties)(screen->pdev, ici->format, ici->imageType,
 260                                                    ici->tiling, ici->usage, ici->flags, &image_props);
 261    if (ret != VK_SUCCESS)
 262       return false;
 263    if (ici->extent.depth > image_props.maxExtent.depth ||
 264        ici->extent.height > image_props.maxExtent.height ||
 265        ici->extent.width > image_props.maxExtent.width)
 266       return false;
 267    if (ici->mipLevels > image_props.maxMipLevels)
 268       return false;
 269    if (ici->arrayLayers > image_props.maxArrayLayers)
 270       return false;
 271    return true;
 272 }
 273
 274 static VkImageUsageFlags
 275 get_image_usage_for_feats(struct zink_screen *screen, VkFormatFeatureFlags feats, const struct pipe_resource *templ, unsigned bind, bool *need_extended)
 276 {
 277    VkImageUsageFlags usage = 0;
 278    bool is_planar = util_format_get_num_planes(templ->format) > 1;
 279    *need_extended = false;
 280
 281    if (bind & ZINK_BIND_TRANSIENT)
 282       usage |= VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT;
 283    else {
 284       /* sadly, gallium doesn't let us know if it'll ever need this, so we have to assume */
 285       if (is_planar || (feats & VK_FORMAT_FEATURE_TRANSFER_SRC_BIT))
 286          usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
 287       if (is_planar || (feats & VK_FORMAT_FEATURE_TRANSFER_DST_BIT))
 288          usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT;
 289       if (feats & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT)
 290          usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
 291
 292       if ((is_planar || (feats & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT)) && (bind & PIPE_BIND_SHADER_IMAGE)) {
 293          assert(templ->nr_samples <= 1 || screen->info.feats.features.shaderStorageImageMultisample);
 294          usage |= VK_IMAGE_USAGE_STORAGE_BIT;
 295       }
 296    }
 297
 298    if (bind & PIPE_BIND_RENDER_TARGET) {
 299       if (feats & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) {
 300          usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 301          if (!(bind & ZINK_BIND_TRANSIENT) && (bind & (PIPE_BIND_LINEAR | PIPE_BIND_SHARED)) != (PIPE_BIND_LINEAR | PIPE_BIND_SHARED))
 302             usage |= VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
 303          if (!(bind & ZINK_BIND_TRANSIENT))
 304             usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
 305       } else {
 306          /* trust that gallium isn't going to give us anything wild */
 307          *need_extended = true;
 308          return 0;
 309       }
 310    } else if ((bind & PIPE_BIND_SAMPLER_VIEW) && !util_format_is_depth_or_stencil(templ->format)) {
 311       if (!(feats & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) {
 312          /* ensure we can u_blitter this later */
 313          *need_extended = true;
 314          return 0;
 315       }
 316       usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 317    }
 318
 319    if (bind & PIPE_BIND_DEPTH_STENCIL) {
 320       if (feats & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT)
 321          usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
 322       else
 323          return 0;
 324       usage |= VK_IMAGE_USAGE_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT;
 325    /* this is unlikely to occur and has been included for completeness */
 326    } else if (bind & PIPE_BIND_SAMPLER_VIEW && !(usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)) {
 327       if (feats & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)
 328          usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 329       else
 330          return 0;
 331    }
 332
 333    if (bind & PIPE_BIND_STREAM_OUTPUT)
 334       usage |= VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
 335
 336    return usage;
 337 }
 338
 339 static VkFormatFeatureFlags
 340 find_modifier_feats(const struct zink_modifier_prop *prop, uint64_t modifier, uint64_t *mod)
 341 {
 342    for (unsigned j = 0; j < prop->drmFormatModifierCount; j++) {
 343       if (prop->pDrmFormatModifierProperties[j].drmFormatModifier == modifier) {
 344          *mod = modifier;
 345          return prop->pDrmFormatModifierProperties[j].drmFormatModifierTilingFeatures;
 346       }
 347    }
 348    return 0;
 349 }
 350
 351 /* If the driver can't do mutable with this ICI, then try again after removing mutable (and
 352  * thus also the list of formats we might might mutate to)
 353  */
 354 static bool
 355 double_check_ici(struct zink_screen *screen, VkImageCreateInfo *ici, VkImageUsageFlags usage, uint64_t *mod)
 356 {
 357     if (!usage)
 358        return false;
 359
 360     const void *pNext = ici->pNext;
 361     ici->usage = usage;
 362     if (check_ici(screen, ici, *mod))
 363        return true;
 364     if (pNext) {
 365        ici->pNext = NULL;
 366        ici->flags &= ~VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 367        if (check_ici(screen, ici, *mod))
 368           return true;
 369        ici->pNext = pNext;
 370        ici->flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 371     }
 372     return false;
 373 }
 374
 375 static VkImageUsageFlags
 376 get_image_usage(struct zink_screen *screen, VkImageCreateInfo *ici, const struct pipe_resource *templ, unsigned bind, unsigned modifiers_count, uint64_t *modifiers, uint64_t *mod)
 377 {
 378    VkImageTiling tiling = ici->tiling;
 379    bool need_extended = false;
 380    *mod = DRM_FORMAT_MOD_INVALID;
 381    if (modifiers_count) {
 382       bool have_linear = false;
 383       const struct zink_modifier_prop *prop = &screen->modifier_props[templ->format];
 384       assert(tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT);
 385       bool found = false;
 386       uint64_t good_mod = 0;
 387       VkImageUsageFlags good_usage = 0;
 388       for (unsigned i = 0; i < modifiers_count; i++) {
 389          if (modifiers[i] == DRM_FORMAT_MOD_LINEAR) {
 390             have_linear = true;
 391             if (!screen->info.have_EXT_image_drm_format_modifier)
 392                break;
 393             continue;
 394          }
 395          VkFormatFeatureFlags feats = find_modifier_feats(prop, modifiers[i], mod);
 396          if (feats) {
 397             VkImageUsageFlags usage = get_image_usage_for_feats(screen, feats, templ, bind, &need_extended);
 398             assert(!need_extended);
 399             if (double_check_ici(screen, ici, usage, mod)) {
 400                if (!found) {
 401                   found = true;
 402                   good_mod = modifiers[i];
 403                   good_usage = usage;
 404                }
 405             } else {
 406                modifiers[i] = DRM_FORMAT_MOD_LINEAR;
 407             }
 408          }
 409       }
 410       if (found) {
 411          *mod = good_mod;
 412          return good_usage;
 413       }
 414       /* only try linear if no other options available */
 415       if (have_linear) {
 416          VkFormatFeatureFlags feats = find_modifier_feats(prop, DRM_FORMAT_MOD_LINEAR, mod);
 417          if (feats) {
 418             VkImageUsageFlags usage = get_image_usage_for_feats(screen, feats, templ, bind, &need_extended);
 419             assert(!need_extended);
 420             if (double_check_ici(screen, ici, usage, mod))
 421                return usage;
 422          }
 423       }
 424    } else {
 425       VkFormatProperties props = screen->format_props[templ->format];
 426       VkFormatFeatureFlags feats = tiling == VK_IMAGE_TILING_LINEAR ? props.linearTilingFeatures : props.optimalTilingFeatures;
 427       if (ici->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT)
 428          feats = UINT32_MAX;
 429       VkImageUsageFlags usage = get_image_usage_for_feats(screen, feats, templ, bind, &need_extended);
 430       if (need_extended) {
 431          ici->flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT | VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 432          feats = UINT32_MAX;
 433          usage = get_image_usage_for_feats(screen, feats, templ, bind, &need_extended);
 434       }
 435       if (double_check_ici(screen, ici, usage, mod))
 436          return usage;
 437       if (util_format_is_depth_or_stencil(templ->format)) {
 438          if (!(templ->bind & PIPE_BIND_DEPTH_STENCIL)) {
 439             usage &= ~VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
 440             if (double_check_ici(screen, ici, usage, mod))
 441                return usage;
 442          }
 443       } else if (!(templ->bind & PIPE_BIND_RENDER_TARGET)) {
 444          usage &= ~VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 445          if (double_check_ici(screen, ici, usage, mod))
 446             return usage;
 447       }
 448    }
 449    *mod = DRM_FORMAT_MOD_INVALID;
 450    return 0;
 451 }
 452
 453 static uint64_t
 454 create_ici(struct zink_screen *screen, VkImageCreateInfo *ici, const struct pipe_resource *templ, bool dmabuf, unsigned bind, unsigned modifiers_count, uint64_t *modifiers, bool *success)
 455 {
 456    ici->sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
 457    /* pNext may already be set */
 458    if (util_format_get_num_planes(templ->format) > 1)
 459       ici->flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
 460    else if (bind & ZINK_BIND_MUTABLE)
 461       ici->flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 462    else
 463       ici->flags = 0;
 464    if (ici->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT)
 465       /* unset VkImageFormatListCreateInfo if mutable */
 466       ici->pNext = NULL;
 467    else if (ici->pNext)
 468       /* add mutable if VkImageFormatListCreateInfo */
 469       ici->flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 470    ici->usage = 0;
 471    ici->queueFamilyIndexCount = 0;
 472
 473    /* assume we're going to be doing some CompressedTexSubImage */
 474    if (util_format_is_compressed(templ->format) && (ici->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT))
 475       ici->flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT;
 476
 477    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 478       ici->flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
 479
 480    bool need_2D = false;
 481    switch (templ->target) {
 482    case PIPE_TEXTURE_1D:
 483    case PIPE_TEXTURE_1D_ARRAY:
 484       if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 485          need_2D |= screen->need_2D_sparse;
 486       if (util_format_is_depth_or_stencil(templ->format))
 487          need_2D |= screen->need_2D_zs;
 488       ici->imageType = need_2D ? VK_IMAGE_TYPE_2D : VK_IMAGE_TYPE_1D;
 489       break;
 490
 491    case PIPE_TEXTURE_CUBE:
 492    case PIPE_TEXTURE_CUBE_ARRAY:
 493    case PIPE_TEXTURE_2D:
 494    case PIPE_TEXTURE_2D_ARRAY:
 495    case PIPE_TEXTURE_RECT:
 496       ici->imageType = VK_IMAGE_TYPE_2D;
 497       break;
 498
 499    case PIPE_TEXTURE_3D:
 500       ici->imageType = VK_IMAGE_TYPE_3D;
 501       ici->flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
 502       if (screen->info.have_EXT_image_2d_view_of_3d)
 503          ici->flags |= VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
 504       break;
 505
 506    case PIPE_BUFFER:
 507       unreachable("PIPE_BUFFER should already be handled");
 508
 509    default:
 510       unreachable("Unknown target");
 511    }
 512
 513    if (screen->info.have_EXT_sample_locations &&
 514        bind & PIPE_BIND_DEPTH_STENCIL &&
 515        util_format_has_depth(util_format_description(templ->format)))
 516       ici->flags |= VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT;
 517
 518    ici->format = zink_get_format(screen, templ->format);
 519    ici->extent.width = templ->width0;
 520    ici->extent.height = templ->height0;
 521    ici->extent.depth = templ->depth0;
 522    ici->mipLevels = templ->last_level + 1;
 523    ici->arrayLayers = MAX2(templ->array_size, 1);
 524    ici->samples = templ->nr_samples ? templ->nr_samples : VK_SAMPLE_COUNT_1_BIT;
 525    ici->tiling = screen->info.have_EXT_image_drm_format_modifier && modifiers_count ?
 526                  VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT :
 527                  bind & (PIPE_BIND_LINEAR | ZINK_BIND_DMABUF) ? VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
 528    /* XXX: does this have perf implications anywhere? hopefully not */
 529    if (ici->samples == VK_SAMPLE_COUNT_1_BIT &&
 530       screen->info.have_EXT_multisampled_render_to_single_sampled &&
 531       ici->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
 532       ici->flags |= VK_IMAGE_CREATE_MULTISAMPLED_RENDER_TO_SINGLE_SAMPLED_BIT_EXT;
 533    ici->sharingMode = VK_SHARING_MODE_EXCLUSIVE;
 534    ici->initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
 535
 536    /* sampleCounts will be set to VK_SAMPLE_COUNT_1_BIT if at least one of the following conditions is true:
 537     * - flags contains VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT
 538     *
 539     * 44.1.1. Supported Sample Counts
 540     */
 541    bool want_cube = ici->samples == 1 &&
 542                     (templ->target == PIPE_TEXTURE_CUBE ||
 543                     templ->target == PIPE_TEXTURE_CUBE_ARRAY ||
 544                     (templ->target == PIPE_TEXTURE_2D_ARRAY && ici->extent.width == ici->extent.height && ici->arrayLayers >= 6));
 545
 546    if (templ->target == PIPE_TEXTURE_CUBE)
 547       ici->arrayLayers *= 6;
 548
 549    if (ici->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
 550       modifiers_count = 0;
 551
 552    bool first = true;
 553    bool tried[2] = {0};
 554    uint64_t mod = DRM_FORMAT_MOD_INVALID;
 555 retry:
 556    while (!ici->usage) {
 557       if (!first) {
 558          switch (ici->tiling) {
 559          case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
 560             ici->tiling = VK_IMAGE_TILING_OPTIMAL;
 561             modifiers_count = 0;
 562             break;
 563          case VK_IMAGE_TILING_OPTIMAL:
 564             ici->tiling = VK_IMAGE_TILING_LINEAR;
 565             break;
 566          case VK_IMAGE_TILING_LINEAR:
 567             if (bind & PIPE_BIND_LINEAR) {
 568                *success = false;
 569                return DRM_FORMAT_MOD_INVALID;
 570             }
 571             ici->tiling = VK_IMAGE_TILING_OPTIMAL;
 572             break;
 573          default:
 574             unreachable("unhandled tiling mode");
 575          }
 576          if (tried[ici->tiling]) {
 577             if (ici->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT) {
 578                *success = false;
 579                return DRM_FORMAT_MOD_INVALID;
 580             }
 581             ici->flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT | VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
 582             tried[0] = false;
 583             tried[1] = false;
 584             first = true;
 585             goto retry;
 586          }
 587       }
 588       ici->usage = get_image_usage(screen, ici, templ, bind, modifiers_count, modifiers, &mod);
 589       first = false;
 590       if (ici->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
 591          tried[ici->tiling] = true;
 592    }
 593    if (want_cube) {
 594       ici->flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
 595       if (get_image_usage(screen, ici, templ, bind, modifiers_count, modifiers, &mod) != ici->usage)
 596          ici->flags &= ~VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
 597    }
 598
 599    *success = true;
 600    return mod;
 601 }
 602
 603 static struct zink_resource_object *
 604 resource_object_create(struct zink_screen *screen, const struct pipe_resource *templ, struct winsys_handle *whandle, bool *linear,
 605                        uint64_t *modifiers, int modifiers_count, const void *loader_private)
 606 {
 607    struct zink_resource_object *obj = CALLOC_STRUCT(zink_resource_object);
 608    unsigned max_level = 0;
 609    if (!obj)
 610       return NULL;
 611    simple_mtx_init(&obj->view_lock, mtx_plain);
 612    util_dynarray_init(&obj->views, NULL);
 613    obj->unordered_read = true;
 614    obj->unordered_write = true;
 615    obj->last_dt_idx = obj->dt_idx = UINT32_MAX; //TODO: unionize
 616
 617    VkMemoryRequirements reqs = {0};
 618    VkMemoryPropertyFlags flags;
 619
 620    /* figure out aux plane count */
 621    if (whandle && whandle->plane >= util_format_get_num_planes(whandle->format))
 622       obj->is_aux = true;
 623    struct pipe_resource *pnext = templ->next;
 624    for (obj->plane_count = 1; pnext; obj->plane_count++, pnext = pnext->next) {
 625       struct zink_resource *next = zink_resource(pnext);
 626       if (!next->obj->is_aux)
 627          break;
 628    }
 629
 630    bool need_dedicated = false;
 631    bool shared = templ->bind & PIPE_BIND_SHARED;
 632 #if !defined(_WIN32)
 633    VkExternalMemoryHandleTypeFlags export_types = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
 634 #else
 635    VkExternalMemoryHandleTypeFlags export_types = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
 636 #endif
 637    unsigned num_planes = util_format_get_num_planes(templ->format);
 638    VkImageAspectFlags plane_aspects[] = {
 639       VK_IMAGE_ASPECT_PLANE_0_BIT,
 640       VK_IMAGE_ASPECT_PLANE_1_BIT,
 641       VK_IMAGE_ASPECT_PLANE_2_BIT,
 642    };
 643    VkExternalMemoryHandleTypeFlags external = 0;
 644    bool needs_export = (templ->bind & (ZINK_BIND_VIDEO | ZINK_BIND_DMABUF)) != 0;
 645    if (whandle) {
 646       if (whandle->type == WINSYS_HANDLE_TYPE_FD || whandle->type == ZINK_EXTERNAL_MEMORY_HANDLE)
 647          needs_export |= true;
 648       else
 649          unreachable("unknown handle type");
 650    }
 651    if (needs_export) {
 652       if (whandle && whandle->type == ZINK_EXTERNAL_MEMORY_HANDLE) {
 653 #if !defined(_WIN32)
 654          external = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
 655 #else
 656          external = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
 657 #endif
 658       } else if (screen->info.have_EXT_external_memory_dma_buf) {
 659          external = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
 660          export_types |= VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
 661       } else {
 662          /* can't export anything, fail early */
 663          return NULL;
 664       }
 665    }
 666
 667    /* we may export WINSYS_HANDLE_TYPE_FD handle which is dma-buf */
 668    if (shared && screen->info.have_EXT_external_memory_dma_buf)
 669       export_types |= VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
 670
 671    pipe_reference_init(&obj->reference, 1);
 672    if (loader_private) {
 673       obj->bo = CALLOC_STRUCT(zink_bo);
 674       obj->transfer_dst = true;
 675       return obj;
 676    } else if (templ->target == PIPE_BUFFER) {
 677       VkBufferCreateInfo bci = create_bci(screen, templ, templ->bind);
 678
 679       if (VKSCR(CreateBuffer)(screen->dev, &bci, NULL, &obj->buffer) != VK_SUCCESS) {
 680          mesa_loge("ZINK: vkCreateBuffer failed");
 681          goto fail1;
 682       }
 683
 684       if (!(templ->bind & (PIPE_BIND_SHADER_IMAGE | ZINK_BIND_DESCRIPTOR))) {
 685          bci.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
 686          if (VKSCR(CreateBuffer)(screen->dev, &bci, NULL, &obj->storage_buffer) != VK_SUCCESS) {
 687             mesa_loge("ZINK: vkCreateBuffer failed");
 688             goto fail2;
 689          }
 690       }
 691
 692       VKSCR(GetBufferMemoryRequirements)(screen->dev, obj->buffer, &reqs);
 693       if (templ->usage == PIPE_USAGE_STAGING)
 694          flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
 695       else if (templ->usage == PIPE_USAGE_STREAM)
 696          flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
 697       else if (templ->usage == PIPE_USAGE_IMMUTABLE)
 698          flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
 699       else
 700          flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
 701       obj->is_buffer = true;
 702       obj->transfer_dst = true;
 703       obj->vkflags = bci.flags;
 704       obj->vkusage = bci.usage;
 705       max_level = 1;
 706    } else {
 707       max_level = templ->last_level + 1;
 708       bool winsys_modifier = (export_types & VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT) && whandle && whandle->modifier != DRM_FORMAT_MOD_INVALID;
 709       uint64_t mods[10];
 710       bool try_modifiers = false;
 711       if ((export_types & VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT) &&
 712           whandle && whandle->modifier == DRM_FORMAT_MOD_INVALID && whandle->stride) {
 713          modifiers = mods;
 714          modifiers_count = screen->modifier_props[templ->format].drmFormatModifierCount;
 715          for (unsigned j = 0; j < modifiers_count; j++)
 716             mods[j] = screen->modifier_props[templ->format].pDrmFormatModifierProperties[j].drmFormatModifier;
 717          if (modifiers_count > 1)
 718             try_modifiers = true;
 719       }
 720       uint64_t *ici_modifiers = winsys_modifier ? &whandle->modifier : modifiers;
 721       unsigned ici_modifier_count = winsys_modifier ? 1 : modifiers_count;
 722       bool success = false;
 723       VkImageCreateInfo ici;
 724       enum pipe_format srgb = PIPE_FORMAT_NONE;
 725       /* We use modifiers as a proxy for "this surface is used as a window system render target".
 726        * For winsys, we need to be able to mutate between srgb and linear, but we don't need general
 727        * image view/shader image format compatibility (that path means losing fast clears or compression on some hardware).
 728        */
 729       if (ici_modifier_count) {
 730          srgb = util_format_is_srgb(templ->format) ? util_format_linear(templ->format) : util_format_srgb(templ->format);
 731          /* why do these helpers have different default return values? */
 732          if (srgb == templ->format)
 733             srgb = PIPE_FORMAT_NONE;
 734       }
 735       VkFormat formats[2];
 736       VkImageFormatListCreateInfo format_list;
 737       if (srgb) {
 738          format_list.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO;
 739          format_list.pNext = NULL;
 740          format_list.viewFormatCount = 2;
 741          format_list.pViewFormats = formats;
 742
 743          formats[0] = zink_get_format(screen, templ->format);
 744          formats[1] = zink_get_format(screen, srgb);
 745          ici.pNext = &format_list;
 746       } else {
 747          ici.pNext = NULL;
 748       }
 749       uint64_t mod = create_ici(screen, &ici, templ, external == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
 750                                 templ->bind, ici_modifier_count, ici_modifiers, &success);
 751       if (ici.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && srgb &&
 752           util_format_get_nr_components(srgb) == 4 &&
 753           !(ici.flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT)) {
 754          mesa_loge("zink: refusing to create possibly-srgb dmabuf due to missing driver support: %s not supported!", util_format_name(srgb));
 755          goto fail1;
 756       }
 757       VkExternalMemoryImageCreateInfo emici;
 758       VkImageDrmFormatModifierExplicitCreateInfoEXT idfmeci;
 759       VkImageDrmFormatModifierListCreateInfoEXT idfmlci;
 760       VkSubresourceLayout plane_layouts[4];
 761       VkSubresourceLayout plane_layout = {
 762          .offset = whandle ? whandle->offset : 0,
 763          .size = 0,
 764          .rowPitch = whandle ? whandle->stride : 0,
 765          .arrayPitch = 0,
 766          .depthPitch = 0,
 767       };
 768       if (!success)
 769          goto fail1;
 770
 771       obj->render_target = (ici.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) != 0;
 772
 773       if (shared || external) {
 774          emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
 775          emici.pNext = ici.pNext;
 776          emici.handleTypes = export_types;
 777          ici.pNext = &emici;
 778
 779          assert(ici.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT || mod != DRM_FORMAT_MOD_INVALID);
 780          if (whandle && ici.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
 781             assert(mod == whandle->modifier || !winsys_modifier);
 782             idfmeci.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT;
 783             idfmeci.pNext = ici.pNext;
 784             idfmeci.drmFormatModifier = mod;
 785
 786             idfmeci.drmFormatModifierPlaneCount = obj->plane_count;
 787             plane_layouts[0] = plane_layout;
 788             pnext = templ->next;
 789             for (unsigned i = 1; i < obj->plane_count; i++, pnext = pnext->next) {
 790                struct zink_resource *next = zink_resource(pnext);
 791                obj->plane_offsets[i] = plane_layouts[i].offset = next->obj->plane_offsets[i];
 792                obj->plane_strides[i] = plane_layouts[i].rowPitch = next->obj->plane_strides[i];
 793                plane_layouts[i].size = 0;
 794                plane_layouts[i].arrayPitch = 0;
 795                plane_layouts[i].depthPitch = 0;
 796             }
 797             idfmeci.pPlaneLayouts = plane_layouts;
 798
 799             ici.pNext = &idfmeci;
 800          } else if (ici.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
 801             idfmlci.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT;
 802             idfmlci.pNext = ici.pNext;
 803             idfmlci.drmFormatModifierCount = modifiers_count;
 804             idfmlci.pDrmFormatModifiers = modifiers;
 805             ici.pNext = &idfmlci;
 806          } else if (ici.tiling == VK_IMAGE_TILING_OPTIMAL) {
 807             shared = false;
 808          }
 809       }
 810
 811       if (linear)
 812          *linear = ici.tiling == VK_IMAGE_TILING_LINEAR;
 813
 814       if (ici.usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
 815          obj->transfer_dst = true;
 816
 817 #if defined(ZINK_USE_DMABUF) && !defined(_WIN32)
 818       if (obj->is_aux) {
 819          obj->modifier = mod;
 820          obj->modifier_aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT << whandle->plane;
 821          obj->plane_offsets[whandle->plane] = whandle->offset;
 822          obj->plane_strides[whandle->plane] = whandle->stride;
 823          obj->handle = os_dupfd_cloexec(whandle->handle);
 824          if (obj->handle < 0) {
 825             mesa_loge("ZINK: failed to dup dmabuf fd: %s\n", strerror(errno));
 826             goto fail1;
 827          }
 828          return obj;
 829       }
 830 #endif
 831
 832       VkFormatFeatureFlags feats = 0;
 833       switch (ici.tiling) {
 834       case VK_IMAGE_TILING_LINEAR:
 835          feats = screen->format_props[templ->format].linearTilingFeatures;
 836          break;
 837       case VK_IMAGE_TILING_OPTIMAL:
 838          feats = screen->format_props[templ->format].optimalTilingFeatures;
 839          break;
 840       case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
 841          feats = VK_FORMAT_FEATURE_FLAG_BITS_MAX_ENUM;
 842          /*
 843             If is tiling then VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT, the value of
 844             imageCreateFormatFeatures is found by calling vkGetPhysicalDeviceFormatProperties2
 845             with VkImageFormatProperties::format equal to VkImageCreateInfo::format and with
 846             VkDrmFormatModifierPropertiesListEXT chained into VkImageFormatProperties2; by
 847             collecting all members of the returned array
 848             VkDrmFormatModifierPropertiesListEXT::pDrmFormatModifierProperties
 849             whose drmFormatModifier belongs to imageCreateDrmFormatModifiers; and by taking the bitwise
 850             intersection, over the collected array members, of drmFormatModifierTilingFeatures.
 851             (The resultant imageCreateFormatFeatures may be empty).
 852             * -Chapter 12. Resource Creation
 853           */
 854          for (unsigned i = 0; i < screen->modifier_props[templ->format].drmFormatModifierCount; i++)
 855             feats &= screen->modifier_props[templ->format].pDrmFormatModifierProperties[i].drmFormatModifierTilingFeatures;
 856          break;
 857       default:
 858          unreachable("unknown tiling");
 859       }
 860       obj->vkfeats = feats;
 861       if (util_format_is_yuv(templ->format)) {
 862          if (feats & VK_FORMAT_FEATURE_DISJOINT_BIT)
 863             ici.flags |= VK_IMAGE_CREATE_DISJOINT_BIT;
 864          VkSamplerYcbcrConversionCreateInfo sycci = {0};
 865          sycci.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO;
 866          sycci.pNext = NULL;
 867          sycci.format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM;
 868          sycci.ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709;
 869          sycci.ycbcrRange = VK_SAMPLER_YCBCR_RANGE_ITU_FULL;
 870          sycci.components.r = VK_COMPONENT_SWIZZLE_IDENTITY;
 871          sycci.components.g = VK_COMPONENT_SWIZZLE_IDENTITY;
 872          sycci.components.b = VK_COMPONENT_SWIZZLE_IDENTITY;
 873          sycci.components.a = VK_COMPONENT_SWIZZLE_IDENTITY;
 874          if (!feats || (feats & VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT)) {
 875             sycci.xChromaOffset = VK_CHROMA_LOCATION_COSITED_EVEN;
 876             sycci.yChromaOffset = VK_CHROMA_LOCATION_COSITED_EVEN;
 877          } else {
 878             assert(feats & VK_FORMAT_FEATURE_MIDPOINT_CHROMA_SAMPLES_BIT);
 879             sycci.xChromaOffset = VK_CHROMA_LOCATION_MIDPOINT;
 880             sycci.yChromaOffset = VK_CHROMA_LOCATION_MIDPOINT;
 881          }
 882          sycci.chromaFilter = VK_FILTER_LINEAR;
 883          sycci.forceExplicitReconstruction = VK_FALSE;
 884          VkResult res = VKSCR(CreateSamplerYcbcrConversion)(screen->dev, &sycci, NULL, &obj->sampler_conversion);
 885          if (res != VK_SUCCESS) {
 886             mesa_loge("ZINK: vkCreateSamplerYcbcrConversion failed");
 887             goto fail1;
 888          }
 889       } else if (whandle) {
 890          obj->plane_strides[whandle->plane] = whandle->stride;
 891       }
 892
 893       VkResult result = VKSCR(CreateImage)(screen->dev, &ici, NULL, &obj->image);
 894       if (result != VK_SUCCESS) {
 895          if (try_modifiers) {
 896             for (unsigned i = 0; i < modifiers_count; i++) {
 897                if (modifiers[i] == mod)
 898                   continue;
 899                idfmeci.drmFormatModifier = modifiers[i];
 900                result = VKSCR(CreateImage)(screen->dev, &ici, NULL, &obj->image);
 901                if (result == VK_SUCCESS)
 902                   break;
 903             }
 904          }
 905       }
 906       if (result != VK_SUCCESS) {
 907          mesa_loge("ZINK: vkCreateImage failed (%s)", vk_Result_to_str(result));
 908          goto fail1;
 909       }
 910
 911       if (ici.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
 912          VkImageDrmFormatModifierPropertiesEXT modprops = {0};
 913          modprops.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT;
 914          result = VKSCR(GetImageDrmFormatModifierPropertiesEXT)(screen->dev, obj->image, &modprops);
 915          if (result != VK_SUCCESS) {
 916             mesa_loge("ZINK: vkGetImageDrmFormatModifierPropertiesEXT failed");
 917             goto fail1;
 918          }
 919          obj->modifier = modprops.drmFormatModifier;
 920          unsigned num_dmabuf_planes = screen->base.get_dmabuf_modifier_planes(&screen->base, obj->modifier, templ->format);
 921          obj->modifier_aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
 922          if (num_dmabuf_planes > 1)
 923             obj->modifier_aspect |= VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
 924          if (num_dmabuf_planes > 2)
 925             obj->modifier_aspect |= VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
 926          if (num_dmabuf_planes > 3)
 927             obj->modifier_aspect |= VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
 928          assert(num_dmabuf_planes <= 4);
 929       }
 930
 931       if (VKSCR(GetImageMemoryRequirements2)) {
 932          VkMemoryRequirements2 req2;
 933          req2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2;
 934          VkImageMemoryRequirementsInfo2 info2;
 935          info2.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2;
 936          info2.pNext = NULL;
 937          info2.image = obj->image;
 938          VkMemoryDedicatedRequirements ded;
 939          ded.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS;
 940          ded.pNext = NULL;
 941          req2.pNext = &ded;
 942          VkImagePlaneMemoryRequirementsInfo plane;
 943          plane.sType = VK_STRUCTURE_TYPE_IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO;
 944          plane.pNext = NULL;
 945          if (num_planes > 1)
 946             info2.pNext = &plane;
 947          unsigned offset = 0;
 948          for (unsigned i = 0; i < num_planes; i++) {
 949             assert(i < ARRAY_SIZE(plane_aspects));
 950             plane.planeAspect = plane_aspects[i];
 951             VKSCR(GetImageMemoryRequirements2)(screen->dev, &info2, &req2);
 952             if (!i)
 953                reqs.alignment = req2.memoryRequirements.alignment;
 954             obj->plane_offsets[i] = offset;
 955             offset += req2.memoryRequirements.size;
 956             reqs.size += req2.memoryRequirements.size;
 957             reqs.memoryTypeBits |= req2.memoryRequirements.memoryTypeBits;
 958             need_dedicated |= ded.prefersDedicatedAllocation || ded.requiresDedicatedAllocation;
 959          }
 960       } else {
 961          VKSCR(GetImageMemoryRequirements)(screen->dev, obj->image, &reqs);
 962       }
 963       if (templ->usage == PIPE_USAGE_STAGING && ici.tiling == VK_IMAGE_TILING_LINEAR)
 964         flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
 965       else
 966         flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
 967
 968       obj->vkflags = ici.flags;
 969       obj->vkusage = ici.usage;
 970    }
 971    obj->alignment = reqs.alignment;
 972
 973    if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC)
 974       flags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
 975    else if (!(flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
 976             templ->usage == PIPE_USAGE_STAGING)
 977       flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
 978
 979    if (templ->bind & ZINK_BIND_TRANSIENT)
 980       flags |= VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT;
 981
 982    VkMemoryAllocateInfo mai;
 983    enum zink_alloc_flag aflags = templ->flags & PIPE_RESOURCE_FLAG_SPARSE ? ZINK_ALLOC_SPARSE : 0;
 984    mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
 985    mai.pNext = NULL;
 986    mai.allocationSize = reqs.size;
 987    enum zink_heap heap = zink_heap_from_domain_flags(flags, aflags);
 988    mai.memoryTypeIndex = zink_mem_type_idx_from_bits(screen, heap, reqs.memoryTypeBits);
 989    if (mai.memoryTypeIndex == UINT32_MAX) {
 990       /* not valid based on reqs; demote to more compatible type */
 991       switch (heap) {
 992       case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
 993          heap = ZINK_HEAP_DEVICE_LOCAL;
 994          break;
 995       case ZINK_HEAP_HOST_VISIBLE_CACHED:
 996          heap = ZINK_HEAP_HOST_VISIBLE_COHERENT;
 997          break;
 998       default:
 999          break;
1000       }
1001       mai.memoryTypeIndex = zink_mem_type_idx_from_bits(screen, heap, reqs.memoryTypeBits);
1002       assert(mai.memoryTypeIndex != UINT32_MAX);
1003    }
1004    assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
1005
1006    VkMemoryDedicatedAllocateInfo ded_alloc_info = {
1007       .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
1008       .pNext = mai.pNext,
1009       .image = obj->image,
1010       .buffer = VK_NULL_HANDLE,
1011    };
1012
1013    if (screen->info.have_KHR_dedicated_allocation && need_dedicated) {
1014       ded_alloc_info.pNext = mai.pNext;
1015       mai.pNext = &ded_alloc_info;
1016    }
1017
1018    VkExportMemoryAllocateInfo emai;
1019    if ((templ->bind & ZINK_BIND_VIDEO) || ((templ->bind & PIPE_BIND_SHARED) && shared) || (templ->bind & ZINK_BIND_DMABUF)) {
1020       emai.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
1021       emai.handleTypes = export_types;
1022
1023       emai.pNext = mai.pNext;
1024       mai.pNext = &emai;
1025       obj->exportable = true;
1026    }
1027
1028 #ifdef ZINK_USE_DMABUF
1029
1030 #if !defined(_WIN32)
1031    VkImportMemoryFdInfoKHR imfi = {
1032       VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
1033       NULL,
1034    };
1035
1036    if (whandle) {
1037       imfi.pNext = NULL;
1038       imfi.handleType = external;
1039       imfi.fd = os_dupfd_cloexec(whandle->handle);
1040       if (imfi.fd < 0) {
1041          mesa_loge("ZINK: failed to dup dmabuf fd: %s\n", strerror(errno));
1042          goto fail1;
1043       }
1044
1045       imfi.pNext = mai.pNext;
1046       mai.pNext = &imfi;
1047    }
1048 #else
1049    VkImportMemoryWin32HandleInfoKHR imfi = {
1050       VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR,
1051       NULL,
1052    };
1053
1054    if (whandle) {
1055       HANDLE source_target = GetCurrentProcess();
1056       HANDLE out_handle;
1057
1058       bool result = DuplicateHandle(source_target, whandle->handle, source_target, &out_handle, 0, false, DUPLICATE_SAME_ACCESS);
1059
1060       if (!result || !out_handle) {
1061          mesa_loge("ZINK: failed to DuplicateHandle with winerr: %08x\n", (int)GetLastError());
1062          goto fail1;
1063       }
1064
1065       imfi.pNext = NULL;
1066       imfi.handleType = external;
1067       imfi.handle = out_handle;
1068
1069       imfi.pNext = mai.pNext;
1070       mai.pNext = &imfi;
1071    }
1072 #endif
1073
1074 #endif
1075
1076    unsigned alignment = MAX2(reqs.alignment, 256);
1077    if (templ->usage == PIPE_USAGE_STAGING && obj->is_buffer)
1078       alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
1079    obj->alignment = alignment;
1080 retry:
1081    obj->bo = zink_bo(zink_bo_create(screen, reqs.size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext));
1082    if (!obj->bo) {
1083       if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) {
1084          if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC)
1085             heap = ZINK_HEAP_HOST_VISIBLE_COHERENT;
1086          else
1087             heap = ZINK_HEAP_DEVICE_LOCAL;
1088          goto retry;
1089       }
1090       goto fail2;
1091    }
1092    if (aflags == ZINK_ALLOC_SPARSE) {
1093       obj->size = templ->width0;
1094    } else {
1095       obj->offset = zink_bo_get_offset(obj->bo);
1096       obj->size = zink_bo_get_size(obj->bo);
1097    }
1098
1099    obj->coherent = screen->info.mem_props.memoryTypes[obj->bo->base.placement].propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
1100    if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE)) {
1101       obj->host_visible = screen->info.mem_props.memoryTypes[obj->bo->base.placement].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
1102    }
1103
1104    if (templ->target == PIPE_BUFFER) {
1105       if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE)) {
1106          if (VKSCR(BindBufferMemory)(screen->dev, obj->buffer, zink_bo_get_mem(obj->bo), obj->offset) != VK_SUCCESS) {
1107             mesa_loge("ZINK: vkBindBufferMemory failed");
1108             goto fail3;
1109          }
1110          if (obj->storage_buffer && VKSCR(BindBufferMemory)(screen->dev, obj->storage_buffer, zink_bo_get_mem(obj->bo), obj->offset) != VK_SUCCESS) {
1111             mesa_loge("ZINK: vkBindBufferMemory failed");
1112             goto fail3;
1113          }
1114       }
1115    } else {
1116       if (num_planes > 1) {
1117          VkBindImageMemoryInfo infos[3];
1118          VkBindImagePlaneMemoryInfo planes[3];
1119          for (unsigned i = 0; i < num_planes; i++) {
1120             infos[i].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
1121             infos[i].image = obj->image;
1122             infos[i].memory = zink_bo_get_mem(obj->bo);
1123             infos[i].memoryOffset = obj->plane_offsets[i];
1124             if (templ->bind & ZINK_BIND_VIDEO) {
1125                infos[i].pNext = &planes[i];
1126                planes[i].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
1127                planes[i].pNext = NULL;
1128                planes[i].planeAspect = plane_aspects[i];
1129             }
1130          }
1131          if (VKSCR(BindImageMemory2)(screen->dev, num_planes, infos) != VK_SUCCESS) {
1132             mesa_loge("ZINK: vkBindImageMemory2 failed");
1133             goto fail3;
1134          }
1135       } else {
1136          if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
1137             if (VKSCR(BindImageMemory)(screen->dev, obj->image, zink_bo_get_mem(obj->bo), obj->offset) != VK_SUCCESS) {
1138                mesa_loge("ZINK: vkBindImageMemory failed");
1139                goto fail3;
1140             }
1141       }
1142    }
1143    for (unsigned i = 0; i < max_level; i++)
1144       util_dynarray_init(&obj->copies[i], NULL);
1145    return obj;
1146
1147 fail3:
1148    zink_bo_unref(screen, obj->bo);
1149
1150 fail2:
1151    if (templ->target == PIPE_BUFFER) {
1152       VKSCR(DestroyBuffer)(screen->dev, obj->buffer, NULL);
1153       VKSCR(DestroyBuffer)(screen->dev, obj->storage_buffer, NULL);
1154    } else
1155       VKSCR(DestroyImage)(screen->dev, obj->image, NULL);
1156 fail1:
1157    FREE(obj);
1158    return NULL;
1159 }
1160
1161 static struct pipe_resource *
1162 resource_create(struct pipe_screen *pscreen,
1163                 const struct pipe_resource *templ,
1164                 struct winsys_handle *whandle,
1165                 unsigned external_usage,
1166                 const uint64_t *modifiers, int modifiers_count,
1167                 const void *loader_private)
1168 {
1169    struct zink_screen *screen = zink_screen(pscreen);
1170    struct zink_resource *res = CALLOC_STRUCT_CL(zink_resource);
1171
1172    if (modifiers_count > 0 && screen->info.have_EXT_image_drm_format_modifier) {
1173       /* for rebinds */
1174       res->modifiers_count = modifiers_count;
1175       res->modifiers = mem_dup(modifiers, modifiers_count * sizeof(uint64_t));
1176       if (!res->modifiers) {
1177          FREE_CL(res);
1178          return NULL;
1179       }
1180    }
1181
1182    res->base.b = *templ;
1183
1184    bool allow_cpu_storage = (templ->target == PIPE_BUFFER) &&
1185                             (templ->width0 < 0x1000);
1186    threaded_resource_init(&res->base.b, allow_cpu_storage);
1187    pipe_reference_init(&res->base.b.reference, 1);
1188    res->base.b.screen = pscreen;
1189
1190    bool linear = false;
1191    struct pipe_resource templ2 = *templ;
1192    if (templ2.flags & PIPE_RESOURCE_FLAG_SPARSE)
1193       templ2.bind |= PIPE_BIND_SHADER_IMAGE;
1194    if (screen->faked_e5sparse && templ->format == PIPE_FORMAT_R9G9B9E5_FLOAT) {
1195       templ2.flags &= ~PIPE_RESOURCE_FLAG_SPARSE;
1196       res->base.b.flags &= ~PIPE_RESOURCE_FLAG_SPARSE;
1197    }
1198    res->obj = resource_object_create(screen, &templ2, whandle, &linear, res->modifiers, res->modifiers_count, loader_private);
1199    if (!res->obj) {
1200       free(res->modifiers);
1201       FREE_CL(res);
1202       return NULL;
1203    }
1204
1205    res->internal_format = templ->format;
1206    if (templ->target == PIPE_BUFFER) {
1207       util_range_init(&res->valid_buffer_range);
1208       res->base.b.bind |= PIPE_BIND_SHADER_IMAGE;
1209       if (!screen->resizable_bar && templ->width0 >= 8196) {
1210          /* We don't want to evict buffers from VRAM by mapping them for CPU access,
1211           * because they might never be moved back again. If a buffer is large enough,
1212           * upload data by copying from a temporary GTT buffer. 8K might not seem much,
1213           * but there can be 100000 buffers.
1214           *
1215           * This tweak improves performance for viewperf.
1216           */
1217          res->base.b.flags |= PIPE_RESOURCE_FLAG_DONT_MAP_DIRECTLY;
1218       }
1219       if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB)
1220          zink_resource_get_address(screen, res);
1221    } else {
1222       if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
1223          res->base.b.bind |= PIPE_BIND_SHADER_IMAGE;
1224       if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE) {
1225          uint32_t count = 1;
1226          VKSCR(GetImageSparseMemoryRequirements)(screen->dev, res->obj->image, &count, &res->sparse);
1227          res->base.b.nr_sparse_levels = res->sparse.imageMipTailFirstLod;
1228       }
1229       res->format = zink_get_format(screen, templ->format);
1230       if (templ->target == PIPE_TEXTURE_1D || templ->target == PIPE_TEXTURE_1D_ARRAY) {
1231          res->need_2D = (screen->need_2D_zs && util_format_is_depth_or_stencil(templ->format)) ||
1232                         (screen->need_2D_sparse && (templ->flags & PIPE_RESOURCE_FLAG_SPARSE));
1233       }
1234       res->dmabuf_acquire = whandle && whandle->type == WINSYS_HANDLE_TYPE_FD;
1235       res->dmabuf = res->dmabuf_acquire = whandle && whandle->type == WINSYS_HANDLE_TYPE_FD;
1236       res->layout = res->dmabuf_acquire ? VK_IMAGE_LAYOUT_PREINITIALIZED : VK_IMAGE_LAYOUT_UNDEFINED;
1237       res->linear = linear;
1238       res->aspect = aspect_from_format(templ->format);
1239    }
1240
1241    if (loader_private) {
1242       if (templ->bind & PIPE_BIND_DISPLAY_TARGET) {
1243          /* backbuffer */
1244          res->obj->dt = zink_kopper_displaytarget_create(screen,
1245                                                          res->base.b.bind,
1246                                                          res->base.b.format,
1247                                                          templ->width0,
1248                                                          templ->height0,
1249                                                          64, loader_private,
1250                                                          &res->dt_stride);
1251          assert(res->obj->dt);
1252       } else {
1253          /* frontbuffer */
1254          struct zink_resource *back = (void*)loader_private;
1255          struct kopper_displaytarget *cdt = back->obj->dt;
1256          cdt->refcount++;
1257          assert(back->obj->dt);
1258          res->obj->dt = back->obj->dt;
1259       }
1260       struct kopper_displaytarget *cdt = res->obj->dt;
1261       if (zink_kopper_has_srgb(cdt))
1262          res->obj->vkflags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
1263       if (cdt->swapchain->scci.flags == VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR)
1264          res->obj->vkflags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
1265       res->obj->vkusage = cdt->swapchain->scci.imageUsage;
1266       res->base.b.bind |= PIPE_BIND_DISPLAY_TARGET;
1267       res->linear = false;
1268       res->swapchain = true;
1269    }
1270    if (!res->obj->host_visible)
1271       res->base.b.flags |= PIPE_RESOURCE_FLAG_DONT_MAP_DIRECTLY;
1272    if (res->obj->is_buffer) {
1273       res->base.buffer_id_unique = util_idalloc_mt_alloc(&screen->buffer_ids);
1274       _mesa_hash_table_init(&res->bufferview_cache, NULL, NULL, equals_bvci);
1275       simple_mtx_init(&res->bufferview_mtx, mtx_plain);
1276    } else {
1277       _mesa_hash_table_init(&res->surface_cache, NULL, NULL, equals_ivci);
1278       simple_mtx_init(&res->surface_mtx, mtx_plain);
1279    }
1280    if (res->obj->exportable)
1281       res->base.b.bind |= ZINK_BIND_DMABUF;
1282    return &res->base.b;
1283 }
1284
1285 static struct pipe_resource *
1286 zink_resource_create(struct pipe_screen *pscreen,
1287                      const struct pipe_resource *templ)
1288 {
1289    return resource_create(pscreen, templ, NULL, 0, NULL, 0, NULL);
1290 }
1291
1292 static struct pipe_resource *
1293 zink_resource_create_with_modifiers(struct pipe_screen *pscreen, const struct pipe_resource *templ,
1294                                     const uint64_t *modifiers, int modifiers_count)
1295 {
1296    return resource_create(pscreen, templ, NULL, 0, modifiers, modifiers_count, NULL);
1297 }
1298
1299 static struct pipe_resource *
1300 zink_resource_create_drawable(struct pipe_screen *pscreen,
1301                               const struct pipe_resource *templ,
1302                               const void *loader_private)
1303 {
1304    return resource_create(pscreen, templ, NULL, 0, NULL, 0, loader_private);
1305 }
1306
1307 static bool
1308 add_resource_bind(struct zink_context *ctx, struct zink_resource *res, unsigned bind)
1309 {
1310    struct zink_screen *screen = zink_screen(ctx->base.screen);
1311    assert((res->base.b.bind & bind) == 0);
1312    zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0);
1313    res->base.b.bind |= bind;
1314    struct zink_resource_object *old_obj = res->obj;
1315    if (bind & ZINK_BIND_DMABUF && !res->modifiers_count && screen->info.have_EXT_image_drm_format_modifier) {
1316       res->modifiers_count = 1;
1317       res->modifiers = malloc(res->modifiers_count * sizeof(uint64_t));
1318       res->modifiers[0] = DRM_FORMAT_MOD_LINEAR;
1319    }
1320    struct zink_resource_object *new_obj = resource_object_create(screen, &res->base.b, NULL, &res->linear, res->modifiers, res->modifiers_count, NULL);
1321    if (!new_obj) {
1322       debug_printf("new backing resource alloc failed!\n");
1323       res->base.b.bind &= ~bind;
1324       return false;
1325    }
1326    struct zink_resource staging = *res;
1327    staging.obj = old_obj;
1328    staging.all_binds = 0;
1329    res->layout = VK_IMAGE_LAYOUT_UNDEFINED;
1330    res->obj->access = 0;
1331    res->obj->access_stage = 0;
1332    res->obj = new_obj;
1333    for (unsigned i = 0; i <= res->base.b.last_level; i++) {
1334       struct pipe_box box = {0, 0, 0,
1335                              u_minify(res->base.b.width0, i),
1336                              u_minify(res->base.b.height0, i), res->base.b.array_size};
1337       box.depth = util_num_layers(&res->base.b, i);
1338       ctx->base.resource_copy_region(&ctx->base, &res->base.b, i, 0, 0, 0, &staging.base.b, i, &box);
1339    }
1340    zink_resource_object_reference(screen, &old_obj, NULL);
1341    return true;
1342 }
1343
1344 static bool
1345 zink_resource_get_param(struct pipe_screen *pscreen, struct pipe_context *pctx,
1346                         struct pipe_resource *pres,
1347                         unsigned plane,
1348                         unsigned layer,
1349                         unsigned level,
1350                         enum pipe_resource_param param,
1351                         unsigned handle_usage,
1352                         uint64_t *value)
1353 {
1354    struct zink_screen *screen = zink_screen(pscreen);
1355    struct zink_resource *res = zink_resource(pres);
1356    struct zink_resource_object *obj = res->obj;
1357    struct winsys_handle whandle;
1358    VkImageAspectFlags aspect;
1359    if (obj->modifier_aspect) {
1360       switch (plane) {
1361       case 0:
1362          aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
1363          break;
1364       case 1:
1365          aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
1366          break;
1367       case 2:
1368          aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
1369          break;
1370       case 3:
1371          aspect = VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
1372          break;
1373       default:
1374          unreachable("how many planes you got in this thing?");
1375       }
1376    } else if (res->obj->sampler_conversion) {
1377       aspect = VK_IMAGE_ASPECT_PLANE_0_BIT;
1378    } else {
1379       aspect = res->aspect;
1380    }
1381    switch (param) {
1382    case PIPE_RESOURCE_PARAM_NPLANES:
1383       if (screen->info.have_EXT_image_drm_format_modifier)
1384          *value = screen->base.get_dmabuf_modifier_planes(&screen->base, obj->modifier, res->internal_format);
1385       else
1386          *value = 1;
1387       break;
1388
1389    case PIPE_RESOURCE_PARAM_STRIDE: {
1390       VkImageSubresource sub_res = {0};
1391       VkSubresourceLayout sub_res_layout = {0};
1392
1393       sub_res.aspectMask = aspect;
1394
1395       VKSCR(GetImageSubresourceLayout)(screen->dev, obj->image, &sub_res, &sub_res_layout);
1396
1397       *value = sub_res_layout.rowPitch;
1398       break;
1399    }
1400
1401    case PIPE_RESOURCE_PARAM_OFFSET: {
1402          VkImageSubresource isr = {
1403             aspect,
1404             level,
1405             layer
1406          };
1407          VkSubresourceLayout srl;
1408          VKSCR(GetImageSubresourceLayout)(screen->dev, obj->image, &isr, &srl);
1409          *value = srl.offset;
1410          break;
1411    }
1412
1413    case PIPE_RESOURCE_PARAM_MODIFIER: {
1414       *value = obj->modifier;
1415       break;
1416    }
1417
1418    case PIPE_RESOURCE_PARAM_LAYER_STRIDE: {
1419          VkImageSubresource isr = {
1420             aspect,
1421             level,
1422             layer
1423          };
1424          VkSubresourceLayout srl;
1425          VKSCR(GetImageSubresourceLayout)(screen->dev, obj->image, &isr, &srl);
1426          if (res->base.b.target == PIPE_TEXTURE_3D)
1427             *value = srl.depthPitch;
1428          else
1429             *value = srl.arrayPitch;
1430          break;
1431    }
1432
1433       return false;
1434    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS:
1435    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
1436    case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD: {
1437 #ifdef ZINK_USE_DMABUF
1438       memset(&whandle, 0, sizeof(whandle));
1439       if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED)
1440          whandle.type = WINSYS_HANDLE_TYPE_SHARED;
1441       if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS)
1442          whandle.type = WINSYS_HANDLE_TYPE_KMS;
1443       else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD)
1444          whandle.type = WINSYS_HANDLE_TYPE_FD;
1445
1446       if (!pscreen->resource_get_handle(pscreen, pctx, pres, &whandle, handle_usage))
1447          return false;
1448
1449 #ifdef _WIN32
1450       *value = (uintptr_t)whandle.handle;
1451 #else
1452       *value = whandle.handle;
1453 #endif
1454       break;
1455 #else
1456       (void)whandle;
1457       return false;
1458 #endif
1459    }
1460    }
1461    return true;
1462 }
1463
1464 static bool
1465 zink_resource_get_handle(struct pipe_screen *pscreen,
1466                          struct pipe_context *context,
1467                          struct pipe_resource *tex,
1468                          struct winsys_handle *whandle,
1469                          unsigned usage)
1470 {
1471    if (tex->target == PIPE_BUFFER)
1472       tc_buffer_disable_cpu_storage(tex);
1473    if (whandle->type == WINSYS_HANDLE_TYPE_FD || whandle->type == WINSYS_HANDLE_TYPE_KMS) {
1474 #ifdef ZINK_USE_DMABUF
1475       struct zink_resource *res = zink_resource(tex);
1476       struct zink_screen *screen = zink_screen(pscreen);
1477       struct zink_resource_object *obj = res->obj;
1478
1479 #if !defined(_WIN32)
1480       if (whandle->type == WINSYS_HANDLE_TYPE_KMS && screen->drm_fd == -1) {
1481          whandle->handle = -1;
1482       } else {
1483          if (!res->obj->exportable) {
1484             assert(!res->all_binds); //TODO handle if problematic
1485             assert(!zink_resource_usage_is_unflushed(res));
1486             unsigned bind = ZINK_BIND_DMABUF;
1487             if (!(res->base.b.bind & PIPE_BIND_SHARED))
1488                bind |= PIPE_BIND_SHARED;
1489             zink_screen_lock_context(screen);
1490             if (!add_resource_bind(screen->copy_context, res, bind)) {
1491                zink_screen_unlock_context(screen);
1492                return false;
1493             }
1494             p_atomic_inc(&screen->image_rebind_counter);
1495             screen->copy_context->base.flush(&screen->copy_context->base, NULL, 0);
1496             zink_screen_unlock_context(screen);
1497             obj = res->obj;
1498          }
1499
1500          VkMemoryGetFdInfoKHR fd_info = {0};
1501          int fd;
1502          fd_info.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
1503          fd_info.memory = zink_bo_get_mem(obj->bo);
1504          if (whandle->type == WINSYS_HANDLE_TYPE_FD)
1505             fd_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
1506          else
1507             fd_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1508          VkResult result = VKSCR(GetMemoryFdKHR)(screen->dev, &fd_info, &fd);
1509          if (result != VK_SUCCESS) {
1510             mesa_loge("ZINK: vkGetMemoryFdKHR failed");
1511             return false;
1512          }
1513          if (whandle->type == WINSYS_HANDLE_TYPE_KMS) {
1514             uint32_t h;
1515             bool ret = zink_bo_get_kms_handle(screen, obj->bo, fd, &h);
1516             close(fd);
1517             if (!ret)
1518                return false;
1519             fd = h;
1520          }
1521
1522          whandle->handle = fd;
1523       }
1524 #else
1525       VkMemoryGetWin32HandleInfoKHR handle_info = {0};
1526       HANDLE handle;
1527       handle_info.sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR;
1528       //TODO: remove for wsi
1529       handle_info.memory = zink_bo_get_mem(obj->bo);
1530       handle_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
1531       VkResult result = VKSCR(GetMemoryWin32HandleKHR)(screen->dev, &handle_info, &handle);
1532       if (result != VK_SUCCESS)
1533          return false;
1534       whandle->handle = handle;
1535 #endif
1536       uint64_t value;
1537       zink_resource_get_param(pscreen, context, tex, 0, 0, 0, PIPE_RESOURCE_PARAM_MODIFIER, 0, &value);
1538       whandle->modifier = value;
1539       zink_resource_get_param(pscreen, context, tex, 0, 0, 0, PIPE_RESOURCE_PARAM_OFFSET, 0, &value);
1540       whandle->offset = value;
1541       zink_resource_get_param(pscreen, context, tex, 0, 0, 0, PIPE_RESOURCE_PARAM_STRIDE, 0, &value);
1542       whandle->stride = value;
1543 #else
1544       return false;
1545 #endif
1546    }
1547    return true;
1548 }
1549
1550 static struct pipe_resource *
1551 zink_resource_from_handle(struct pipe_screen *pscreen,
1552                  const struct pipe_resource *templ,
1553                  struct winsys_handle *whandle,
1554                  unsigned usage)
1555 {
1556 #ifdef ZINK_USE_DMABUF
1557    if (whandle->modifier != DRM_FORMAT_MOD_INVALID &&
1558        !zink_screen(pscreen)->info.have_EXT_image_drm_format_modifier)
1559       return NULL;
1560
1561    struct pipe_resource templ2 = *templ;
1562    if (templ->format == PIPE_FORMAT_NONE)
1563       templ2.format = whandle->format;
1564
1565    uint64_t modifier = DRM_FORMAT_MOD_INVALID;
1566    int modifier_count = 0;
1567    if (whandle->modifier != DRM_FORMAT_MOD_INVALID) {
1568       modifier = whandle->modifier;
1569       modifier_count = 1;
1570    }
1571    struct pipe_resource *pres = resource_create(pscreen, &templ2, whandle, usage, &modifier, modifier_count, NULL);
1572    if (pres) {
1573       struct zink_resource *res = zink_resource(pres);
1574       if (pres->target != PIPE_BUFFER)
1575          res->valid = true;
1576       else
1577          tc_buffer_disable_cpu_storage(pres);
1578       res->internal_format = whandle->format;
1579    }
1580    return pres;
1581 #else
1582    return NULL;
1583 #endif
1584 }
1585
1586 struct zink_memory_object {
1587    struct pipe_memory_object b;
1588    struct winsys_handle whandle;
1589 };
1590
1591 static struct pipe_memory_object *
1592 zink_memobj_create_from_handle(struct pipe_screen *pscreen, struct winsys_handle *whandle, bool dedicated)
1593 {
1594    struct zink_memory_object *memobj = CALLOC_STRUCT(zink_memory_object);
1595    if (!memobj)
1596       return NULL;
1597    memcpy(&memobj->whandle, whandle, sizeof(struct winsys_handle));
1598    memobj->whandle.type = ZINK_EXTERNAL_MEMORY_HANDLE;
1599
1600 #ifdef ZINK_USE_DMABUF
1601
1602 #if !defined(_WIN32)
1603    memobj->whandle.handle = os_dupfd_cloexec(whandle->handle);
1604 #else
1605    HANDLE source_target = GetCurrentProcess();
1606    HANDLE out_handle;
1607
1608    DuplicateHandle(source_target, whandle->handle, source_target, &out_handle, 0, false, DUPLICATE_SAME_ACCESS);
1609    memobj->whandle.handle = out_handle;
1610
1611 #endif /* _WIN32 */
1612 #endif /* ZINK_USE_DMABUF */
1613
1614    return (struct pipe_memory_object *)memobj;
1615 }
1616
1617 static void
1618 zink_memobj_destroy(struct pipe_screen *pscreen, struct pipe_memory_object *pmemobj)
1619 {
1620 #ifdef ZINK_USE_DMABUF
1621    struct zink_memory_object *memobj = (struct zink_memory_object *)pmemobj;
1622
1623 #if !defined(_WIN32)
1624    close(memobj->whandle.handle);
1625 #else
1626    CloseHandle(memobj->whandle.handle);
1627 #endif /* _WIN32 */
1628 #endif /* ZINK_USE_DMABUF */
1629
1630    FREE(pmemobj);
1631 }
1632
1633 static struct pipe_resource *
1634 zink_resource_from_memobj(struct pipe_screen *pscreen,
1635                           const struct pipe_resource *templ,
1636                           struct pipe_memory_object *pmemobj,
1637                           uint64_t offset)
1638 {
1639    struct zink_memory_object *memobj = (struct zink_memory_object *)pmemobj;
1640
1641    struct pipe_resource *pres = resource_create(pscreen, templ, &memobj->whandle, 0, NULL, 0, NULL);
1642    if (pres) {
1643       if (pres->target != PIPE_BUFFER)
1644          zink_resource(pres)->valid = true;
1645       else
1646          tc_buffer_disable_cpu_storage(pres);
1647    }
1648    return pres;
1649 }
1650
1651 static bool
1652 invalidate_buffer(struct zink_context *ctx, struct zink_resource *res)
1653 {
1654    struct zink_screen *screen = zink_screen(ctx->base.screen);
1655
1656    assert(res->base.b.target == PIPE_BUFFER);
1657
1658    if (res->base.b.flags & PIPE_RESOURCE_FLAG_SPARSE)
1659       return false;
1660
1661    if (res->valid_buffer_range.start > res->valid_buffer_range.end)
1662       return false;
1663
1664    if (res->so_valid)
1665       ctx->dirty_so_targets = true;
1666    /* force counter buffer reset */
1667    res->so_valid = false;
1668
1669    util_range_set_empty(&res->valid_buffer_range);
1670    if (!zink_resource_has_usage(res))
1671       return false;
1672
1673    struct zink_resource_object *new_obj = resource_object_create(screen, &res->base.b, NULL, NULL, NULL, 0, NULL);
1674    if (!new_obj) {
1675       debug_printf("new backing resource alloc failed!\n");
1676       return false;
1677    }
1678    /* this ref must be transferred before rebind or else BOOM */
1679    zink_batch_reference_resource_move(&ctx->batch, res);
1680    res->obj = new_obj;
1681    zink_resource_rebind(ctx, res);
1682    return true;
1683 }
1684
1685
1686 static void
1687 zink_resource_invalidate(struct pipe_context *pctx, struct pipe_resource *pres)
1688 {
1689    if (pres->target == PIPE_BUFFER)
1690       invalidate_buffer(zink_context(pctx), zink_resource(pres));
1691    else {
1692       struct zink_resource *res = zink_resource(pres);
1693       if (res->valid && res->fb_bind_count)
1694          zink_context(pctx)->rp_loadop_changed = true;
1695       res->valid = false;
1696    }
1697 }
1698
1699 static void
1700 zink_transfer_copy_bufimage(struct zink_context *ctx,
1701                             struct zink_resource *dst,
1702                             struct zink_resource *src,
1703                             struct zink_transfer *trans)
1704 {
1705    assert((trans->base.b.usage & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) !=
1706           (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY));
1707
1708    bool buf2img = src->base.b.target == PIPE_BUFFER;
1709
1710    struct pipe_box box = trans->base.b.box;
1711    int x = box.x;
1712    if (buf2img)
1713       box.x = trans->offset;
1714
1715    if (dst->obj->transfer_dst)
1716       zink_copy_image_buffer(ctx, dst, src, trans->base.b.level, buf2img ? x : 0,
1717                               box.y, box.z, trans->base.b.level, &box, trans->base.b.usage);
1718    else
1719       util_blitter_copy_texture(ctx->blitter, &dst->base.b, trans->base.b.level,
1720                                 x, box.y, box.z, &src->base.b,
1721                                 0, &box);
1722 }
1723
1724 ALWAYS_INLINE static void
1725 align_offset_size(const VkDeviceSize alignment, VkDeviceSize *offset, VkDeviceSize *size, VkDeviceSize obj_size)
1726 {
1727    VkDeviceSize align = *offset % alignment;
1728    if (alignment - 1 > *offset)
1729       *offset = 0;
1730    else
1731       *offset -= align, *size += align;
1732    align = alignment - (*size % alignment);
1733    if (*offset + *size + align > obj_size)
1734       *size = obj_size - *offset;
1735    else
1736       *size += align;
1737 }
1738
1739 VkMappedMemoryRange
1740 zink_resource_init_mem_range(struct zink_screen *screen, struct zink_resource_object *obj, VkDeviceSize offset, VkDeviceSize size)
1741 {
1742    assert(obj->size);
1743    align_offset_size(screen->info.props.limits.nonCoherentAtomSize, &offset, &size, obj->size);
1744    VkMappedMemoryRange range = {
1745       VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
1746       NULL,
1747       zink_bo_get_mem(obj->bo),
1748       offset,
1749       size
1750    };
1751    assert(range.size);
1752    return range;
1753 }
1754
1755 static void *
1756 map_resource(struct zink_screen *screen, struct zink_resource *res)
1757 {
1758    assert(res->obj->host_visible);
1759    return zink_bo_map(screen, res->obj->bo);
1760 }
1761
1762 static void
1763 unmap_resource(struct zink_screen *screen, struct zink_resource *res)
1764 {
1765    zink_bo_unmap(screen, res->obj->bo);
1766 }
1767
1768 static struct zink_transfer *
1769 create_transfer(struct zink_context *ctx, struct pipe_resource *pres, unsigned usage, const struct pipe_box *box)
1770 {
1771    struct zink_transfer *trans;
1772
1773    if (usage & PIPE_MAP_THREAD_SAFE)
1774       trans = calloc(1, sizeof(*trans));
1775    else if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
1776       trans = slab_zalloc(&ctx->transfer_pool_unsync);
1777    else
1778       trans = slab_zalloc(&ctx->transfer_pool);
1779    if (!trans)
1780       return NULL;
1781
1782    pipe_resource_reference(&trans->base.b.resource, pres);
1783
1784    trans->base.b.usage = usage;
1785    trans->base.b.box = *box;
1786    return trans;
1787 }
1788
1789 static void
1790 destroy_transfer(struct zink_context *ctx, struct zink_transfer *trans)
1791 {
1792    if (trans->base.b.usage & PIPE_MAP_THREAD_SAFE) {
1793       free(trans);
1794    } else {
1795       /* Don't use pool_transfers_unsync. We are always in the driver
1796        * thread. Freeing an object into a different pool is allowed.
1797        */
1798       slab_free(&ctx->transfer_pool, trans);
1799    }
1800 }
1801
1802 static void *
1803 zink_buffer_map(struct pipe_context *pctx,
1804                     struct pipe_resource *pres,
1805                     unsigned level,
1806                     unsigned usage,
1807                     const struct pipe_box *box,
1808                     struct pipe_transfer **transfer)
1809 {
1810    struct zink_context *ctx = zink_context(pctx);
1811    struct zink_screen *screen = zink_screen(pctx->screen);
1812    struct zink_resource *res = zink_resource(pres);
1813    struct zink_transfer *trans = create_transfer(ctx, pres, usage, box);
1814    if (!trans)
1815       return NULL;
1816
1817    void *ptr = NULL;
1818
1819    if (res->base.is_user_ptr)
1820       usage |= PIPE_MAP_PERSISTENT;
1821
1822    /* See if the buffer range being mapped has never been initialized,
1823     * in which case it can be mapped unsynchronized. */
1824    if (!(usage & (PIPE_MAP_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED)) &&
1825        usage & PIPE_MAP_WRITE && !res->base.is_shared &&
1826        !util_ranges_intersect(&res->valid_buffer_range, box->x, box->x + box->width)) {
1827       usage |= PIPE_MAP_UNSYNCHRONIZED;
1828    }
1829
1830    /* If discarding the entire range, discard the whole resource instead. */
1831    if (usage & PIPE_MAP_DISCARD_RANGE && box->x == 0 && box->width == res->base.b.width0) {
1832       usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
1833    }
1834
1835    /* If a buffer in VRAM is too large and the range is discarded, don't
1836     * map it directly. This makes sure that the buffer stays in VRAM.
1837     */
1838    bool force_discard_range = false;
1839    if (usage & (PIPE_MAP_DISCARD_WHOLE_RESOURCE | PIPE_MAP_DISCARD_RANGE) &&
1840        !(usage & PIPE_MAP_PERSISTENT) &&
1841        res->base.b.flags & PIPE_RESOURCE_FLAG_DONT_MAP_DIRECTLY) {
1842       usage &= ~(PIPE_MAP_DISCARD_WHOLE_RESOURCE | PIPE_MAP_UNSYNCHRONIZED);
1843       usage |= PIPE_MAP_DISCARD_RANGE;
1844       force_discard_range = true;
1845    }
1846
1847    if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE &&
1848        !(usage & (PIPE_MAP_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INVALIDATE))) {
1849       assert(usage & PIPE_MAP_WRITE);
1850
1851       if (invalidate_buffer(ctx, res)) {
1852          /* At this point, the buffer is always idle. */
1853          usage |= PIPE_MAP_UNSYNCHRONIZED;
1854       } else {
1855          /* Fall back to a temporary buffer. */
1856          usage |= PIPE_MAP_DISCARD_RANGE;
1857       }
1858    }
1859
1860    unsigned map_offset = box->x;
1861    if (usage & PIPE_MAP_DISCARD_RANGE &&
1862         (!res->obj->host_visible ||
1863         !(usage & (PIPE_MAP_UNSYNCHRONIZED | PIPE_MAP_PERSISTENT)))) {
1864
1865       /* Check if mapping this buffer would cause waiting for the GPU.
1866        */
1867
1868       if (!res->obj->host_visible || force_discard_range ||
1869           !zink_resource_usage_check_completion(screen, res, ZINK_RESOURCE_ACCESS_RW)) {
1870          /* Do a wait-free write-only transfer using a temporary buffer. */
1871          unsigned offset;
1872
1873          /* If we are not called from the driver thread, we have
1874           * to use the uploader from u_threaded_context, which is
1875           * local to the calling thread.
1876           */
1877          struct u_upload_mgr *mgr;
1878          if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
1879             mgr = ctx->tc->base.stream_uploader;
1880          else
1881             mgr = ctx->base.stream_uploader;
1882          u_upload_alloc(mgr, 0, box->width,
1883                      screen->info.props.limits.minMemoryMapAlignment, &offset,
1884                      (struct pipe_resource **)&trans->staging_res, (void **)&ptr);
1885          res = zink_resource(trans->staging_res);
1886          trans->offset = offset;
1887          usage |= PIPE_MAP_UNSYNCHRONIZED;
1888          ptr = ((uint8_t *)ptr);
1889       } else {
1890          /* At this point, the buffer is always idle (we checked it above). */
1891          usage |= PIPE_MAP_UNSYNCHRONIZED;
1892       }
1893    } else if (usage & PIPE_MAP_DONTBLOCK) {
1894       /* sparse/device-local will always need to wait since it has to copy */
1895       if (!res->obj->host_visible)
1896          goto success;
1897       if (!zink_resource_usage_check_completion(screen, res, ZINK_RESOURCE_ACCESS_WRITE))
1898          goto success;
1899       usage |= PIPE_MAP_UNSYNCHRONIZED;
1900    } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1901               (((usage & PIPE_MAP_READ) && !(usage & PIPE_MAP_PERSISTENT) && res->base.b.usage != PIPE_USAGE_STAGING) || !res->obj->host_visible)) {
1902       assert(!(usage & (TC_TRANSFER_MAP_THREADED_UNSYNC | PIPE_MAP_THREAD_SAFE)));
1903       if (!res->obj->host_visible || !(usage & PIPE_MAP_ONCE)) {
1904 overwrite:
1905          trans->offset = box->x % screen->info.props.limits.minMemoryMapAlignment;
1906          trans->staging_res = pipe_buffer_create(&screen->base, PIPE_BIND_LINEAR, PIPE_USAGE_STAGING, box->width + trans->offset);
1907          if (!trans->staging_res)
1908             goto fail;
1909          struct zink_resource *staging_res = zink_resource(trans->staging_res);
1910          zink_copy_buffer(ctx, staging_res, res, trans->offset, box->x, box->width);
1911          res = staging_res;
1912          usage &= ~PIPE_MAP_UNSYNCHRONIZED;
1913          map_offset = trans->offset;
1914       }
1915    } else if ((usage & PIPE_MAP_UNSYNCHRONIZED) && !res->obj->host_visible) {
1916       trans->offset = box->x % screen->info.props.limits.minMemoryMapAlignment;
1917       trans->staging_res = pipe_buffer_create(&screen->base, PIPE_BIND_LINEAR, PIPE_USAGE_STAGING, box->width + trans->offset);
1918       if (!trans->staging_res)
1919          goto fail;
1920       struct zink_resource *staging_res = zink_resource(trans->staging_res);
1921       res = staging_res;
1922       map_offset = trans->offset;
1923    }
1924
1925    if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
1926       if (usage & PIPE_MAP_WRITE) {
1927          if (!(usage & PIPE_MAP_READ)) {
1928             zink_resource_usage_try_wait(ctx, res, ZINK_RESOURCE_ACCESS_RW);
1929             if (zink_resource_has_unflushed_usage(res))
1930                goto overwrite;
1931          }
1932          zink_resource_usage_wait(ctx, res, ZINK_RESOURCE_ACCESS_RW);
1933       } else
1934          zink_resource_usage_wait(ctx, res, ZINK_RESOURCE_ACCESS_WRITE);
1935       res->obj->access = 0;
1936       res->obj->access_stage = 0;
1937    }
1938
1939    if (!ptr) {
1940       /* if writing to a streamout buffer, ensure synchronization next time it's used */
1941       if (usage & PIPE_MAP_WRITE && res->so_valid) {
1942          ctx->dirty_so_targets = true;
1943          /* force counter buffer reset */
1944          res->so_valid = false;
1945       }
1946       ptr = map_resource(screen, res);
1947       if (!ptr)
1948          goto fail;
1949       ptr = ((uint8_t *)ptr) + map_offset;
1950    }
1951
1952    if (!res->obj->coherent
1953 #if defined(MVK_VERSION)
1954       // Work around for MoltenVk limitation specifically on coherent memory
1955       // MoltenVk returns blank memory ranges when there should be data present
1956       // This is a known limitation of MoltenVK.
1957       // See https://github.com/KhronosGroup/MoltenVK/blob/master/Docs/MoltenVK_Runtime_UserGuide.md#known-moltenvk-limitations
1958
1959        || screen->instance_info.have_MVK_moltenvk
1960 #endif
1961       ) {
1962       VkDeviceSize size = box->width;
1963       VkDeviceSize offset = res->obj->offset + trans->offset;
1964       VkMappedMemoryRange range = zink_resource_init_mem_range(screen, res->obj, offset, size);
1965       if (VKSCR(InvalidateMappedMemoryRanges)(screen->dev, 1, &range) != VK_SUCCESS) {
1966          mesa_loge("ZINK: vkInvalidateMappedMemoryRanges failed");
1967          zink_bo_unmap(screen, res->obj->bo);
1968          goto fail;
1969       }
1970    }
1971    trans->base.b.usage = usage;
1972    if (usage & PIPE_MAP_WRITE)
1973       util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width);
1974    if ((usage & PIPE_MAP_PERSISTENT) && !(usage & PIPE_MAP_COHERENT))
1975       res->obj->persistent_maps++;
1976
1977 success:
1978    *transfer = &trans->base.b;
1979    return ptr;
1980
1981 fail:
1982    destroy_transfer(ctx, trans);
1983    return NULL;
1984 }
1985
1986 static void *
1987 zink_image_map(struct pipe_context *pctx,
1988                   struct pipe_resource *pres,
1989                   unsigned level,
1990                   unsigned usage,
1991                   const struct pipe_box *box,
1992                   struct pipe_transfer **transfer)
1993 {
1994    struct zink_context *ctx = zink_context(pctx);
1995    struct zink_screen *screen = zink_screen(pctx->screen);
1996    struct zink_resource *res = zink_resource(pres);
1997    struct zink_transfer *trans = create_transfer(ctx, pres, usage, box);
1998    if (!trans)
1999       return NULL;
2000
2001    trans->base.b.level = level;
2002    if (zink_is_swapchain(res))
2003       /* this is probably a multi-chain which has already been acquired */
2004       zink_kopper_acquire(ctx, res, 0);
2005
2006    void *ptr;
2007    if (usage & PIPE_MAP_WRITE && !(usage & PIPE_MAP_READ))
2008       /* this is like a blit, so we can potentially dump some clears or maybe we have to  */
2009       zink_fb_clears_apply_or_discard(ctx, pres, zink_rect_from_box(box), false);
2010    else if (usage & PIPE_MAP_READ)
2011       /* if the map region intersects with any clears then we have to apply them */
2012       zink_fb_clears_apply_region(ctx, pres, zink_rect_from_box(box));
2013    if (!res->linear || !res->obj->host_visible) {
2014       enum pipe_format format = pres->format;
2015       if (usage & PIPE_MAP_DEPTH_ONLY)
2016          format = util_format_get_depth_only(pres->format);
2017       else if (usage & PIPE_MAP_STENCIL_ONLY)
2018          format = PIPE_FORMAT_S8_UINT;
2019       trans->base.b.stride = util_format_get_stride(format, box->width);
2020       trans->base.b.layer_stride = util_format_get_2d_size(format,
2021                                                          trans->base.b.stride,
2022                                                          box->height);
2023
2024       struct pipe_resource templ = *pres;
2025       templ.next = NULL;
2026       templ.format = format;
2027       templ.usage = usage & PIPE_MAP_READ ? PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
2028       templ.target = PIPE_BUFFER;
2029       templ.bind = PIPE_BIND_LINEAR;
2030       templ.width0 = trans->base.b.layer_stride * box->depth;
2031       templ.height0 = templ.depth0 = 0;
2032       templ.last_level = 0;
2033       templ.array_size = 1;
2034       templ.flags = 0;
2035
2036       trans->staging_res = zink_resource_create(pctx->screen, &templ);
2037       if (!trans->staging_res)
2038          goto fail;
2039
2040       struct zink_resource *staging_res = zink_resource(trans->staging_res);
2041
2042       if (usage & PIPE_MAP_READ) {
2043          /* force multi-context sync */
2044          if (zink_resource_usage_is_unflushed_write(res))
2045             zink_resource_usage_wait(ctx, res, ZINK_RESOURCE_ACCESS_WRITE);
2046          zink_transfer_copy_bufimage(ctx, staging_res, res, trans);
2047          /* need to wait for rendering to finish */
2048          zink_fence_wait(pctx);
2049       }
2050
2051       ptr = map_resource(screen, staging_res);
2052    } else {
2053       assert(res->linear);
2054       ptr = map_resource(screen, res);
2055       if (!ptr)
2056          goto fail;
2057       if (zink_resource_has_usage(res)) {
2058          if (usage & PIPE_MAP_WRITE)
2059             zink_fence_wait(pctx);
2060          else
2061             zink_resource_usage_wait(ctx, res, ZINK_RESOURCE_ACCESS_WRITE);
2062       }
2063       VkImageSubresource isr = {
2064          res->modifiers ? res->obj->modifier_aspect : res->aspect,
2065          level,
2066          0
2067       };
2068       VkSubresourceLayout srl;
2069       VKSCR(GetImageSubresourceLayout)(screen->dev, res->obj->image, &isr, &srl);
2070       trans->base.b.stride = srl.rowPitch;
2071       if (res->base.b.target == PIPE_TEXTURE_3D)
2072          trans->base.b.layer_stride = srl.depthPitch;
2073       else
2074          trans->base.b.layer_stride = srl.arrayPitch;
2075       trans->offset = srl.offset;
2076       trans->depthPitch = srl.depthPitch;
2077       const struct util_format_description *desc = util_format_description(res->base.b.format);
2078       unsigned offset = srl.offset +
2079                         box->z * srl.depthPitch +
2080                         (box->y / desc->block.height) * srl.rowPitch +
2081                         (box->x / desc->block.width) * (desc->block.bits / 8);
2082       if (!res->obj->coherent) {
2083          VkDeviceSize size = (VkDeviceSize)box->width * box->height * desc->block.bits / 8;
2084          VkMappedMemoryRange range = zink_resource_init_mem_range(screen, res->obj, res->obj->offset + offset, size);
2085          if (VKSCR(FlushMappedMemoryRanges)(screen->dev, 1, &range) != VK_SUCCESS) {
2086             mesa_loge("ZINK: vkFlushMappedMemoryRanges failed");
2087          }
2088       }
2089       ptr = ((uint8_t *)ptr) + offset;
2090    }
2091    if (!ptr)
2092       goto fail;
2093    if (usage & PIPE_MAP_WRITE) {
2094       if (!res->valid && res->fb_bind_count)
2095          ctx->rp_loadop_changed = true;
2096       res->valid = true;
2097    }
2098
2099    if (sizeof(void*) == 4)
2100       trans->base.b.usage |= ZINK_MAP_TEMPORARY;
2101    if ((usage & PIPE_MAP_PERSISTENT) && !(usage & PIPE_MAP_COHERENT))
2102       res->obj->persistent_maps++;
2103
2104    *transfer = &trans->base.b;
2105    return ptr;
2106
2107 fail:
2108    destroy_transfer(ctx, trans);
2109    return NULL;
2110 }
2111
2112 static void
2113 zink_transfer_flush_region(struct pipe_context *pctx,
2114                            struct pipe_transfer *ptrans,
2115                            const struct pipe_box *box)
2116 {
2117    struct zink_context *ctx = zink_context(pctx);
2118    struct zink_resource *res = zink_resource(ptrans->resource);
2119    struct zink_transfer *trans = (struct zink_transfer *)ptrans;
2120
2121    if (trans->base.b.usage & PIPE_MAP_WRITE) {
2122       struct zink_screen *screen = zink_screen(pctx->screen);
2123       struct zink_resource *m = trans->staging_res ? zink_resource(trans->staging_res) :
2124                                                      res;
2125       ASSERTED VkDeviceSize size, src_offset, dst_offset = 0;
2126       if (m->obj->is_buffer) {
2127          size = box->width;
2128          src_offset = box->x + (trans->staging_res ? trans->offset : ptrans->box.x);
2129          dst_offset = box->x + ptrans->box.x;
2130       } else {
2131          size = (VkDeviceSize)box->width * box->height * util_format_get_blocksize(m->base.b.format);
2132          src_offset = trans->offset +
2133                   box->z * trans->depthPitch +
2134                   util_format_get_2d_size(m->base.b.format, trans->base.b.stride, box->y) +
2135                   util_format_get_stride(m->base.b.format, box->x);
2136          assert(src_offset + size <= res->obj->size);
2137       }
2138       if (!m->obj->coherent) {
2139          VkMappedMemoryRange range = zink_resource_init_mem_range(screen, m->obj, m->obj->offset, m->obj->size);
2140          if (VKSCR(FlushMappedMemoryRanges)(screen->dev, 1, &range) != VK_SUCCESS) {
2141             mesa_loge("ZINK: vkFlushMappedMemoryRanges failed");
2142          }
2143       }
2144       if (trans->staging_res) {
2145          struct zink_resource *staging_res = zink_resource(trans->staging_res);
2146
2147          if (ptrans->resource->target == PIPE_BUFFER)
2148             zink_copy_buffer(ctx, res, staging_res, dst_offset, src_offset, size);
2149          else
2150             zink_transfer_copy_bufimage(ctx, res, staging_res, trans);
2151       }
2152    }
2153 }
2154
2155 /* used to determine whether to emit a TRANSFER_DST barrier on copies */
2156 bool
2157 zink_resource_copy_box_intersects(struct zink_resource *res, unsigned level, const struct pipe_box *box)
2158 {
2159    /* if there are no valid copy rects tracked, this needs a barrier */
2160    if (!res->obj->copies_valid)
2161       return true;
2162    /* untracked huge miplevel */
2163    if (level >= ARRAY_SIZE(res->obj->copies))
2164       return true;
2165    struct pipe_box *b = res->obj->copies[level].data;
2166    unsigned num_boxes = util_dynarray_num_elements(&res->obj->copies[level], struct pipe_box);
2167    boolean (*intersect)(const struct pipe_box *, const struct pipe_box *);
2168    /* determine intersection function based on dimensionality */
2169    switch (res->base.b.target) {
2170    case PIPE_BUFFER:
2171    case PIPE_TEXTURE_1D:
2172       intersect = u_box_test_intersection_1d;
2173       break;
2174
2175    case PIPE_TEXTURE_1D_ARRAY:
2176    case PIPE_TEXTURE_2D:
2177       intersect = u_box_test_intersection_2d;
2178       break;
2179
2180    default:
2181       intersect = u_box_test_intersection_3d;
2182       break;
2183    }
2184    /* if any of the tracked boxes intersect with this one, a barrier is needed */
2185    for (unsigned i = 0; i < num_boxes; i++) {
2186       if (intersect(box, b + i))
2187          return true;
2188    }
2189    /* no intersection = no barrier */
2190    return false;
2191 }
2192
2193 /* track a new region for TRANSFER_DST barrier emission */
2194 void
2195 zink_resource_copy_box_add(struct zink_resource *res, unsigned level, const struct pipe_box *box)
2196 {
2197    util_dynarray_append(&res->obj->copies[level], struct pipe_box, *box);
2198    res->obj->copies_valid = true;
2199 }
2200
2201 void
2202 zink_resource_copies_reset(struct zink_resource *res)
2203 {
2204    if (!res->obj->copies_valid)
2205       return;
2206    unsigned max_level = res->base.b.target == PIPE_BUFFER ? 1 : (res->base.b.last_level + 1);
2207    for (unsigned i = 0; i < max_level; i++)
2208       util_dynarray_clear(&res->obj->copies[i]);
2209    res->obj->copies_valid = false;
2210 }
2211
2212 static void
2213 transfer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
2214 {
2215    struct zink_context *ctx = zink_context(pctx);
2216    struct zink_resource *res = zink_resource(ptrans->resource);
2217    struct zink_transfer *trans = (struct zink_transfer *)ptrans;
2218
2219    if (!(trans->base.b.usage & (PIPE_MAP_FLUSH_EXPLICIT | PIPE_MAP_COHERENT))) {
2220       /* flush_region is relative to the mapped region: use only the extents */
2221       struct pipe_box box = ptrans->box;
2222       box.x = box.y = box.z = 0;
2223       zink_transfer_flush_region(pctx, ptrans, &box);
2224    }
2225
2226    if ((trans->base.b.usage & PIPE_MAP_PERSISTENT) && !(trans->base.b.usage & PIPE_MAP_COHERENT))
2227       res->obj->persistent_maps--;
2228
2229    if (trans->staging_res)
2230       pipe_resource_reference(&trans->staging_res, NULL);
2231    pipe_resource_reference(&trans->base.b.resource, NULL);
2232
2233    destroy_transfer(ctx, trans);
2234 }
2235
2236 static void
2237 do_transfer_unmap(struct zink_screen *screen, struct zink_transfer *trans)
2238 {
2239    struct zink_resource *res = zink_resource(trans->staging_res);
2240    if (!res)
2241       res = zink_resource(trans->base.b.resource);
2242    unmap_resource(screen, res);
2243 }
2244
2245 static void
2246 zink_buffer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
2247 {
2248    struct zink_screen *screen = zink_screen(pctx->screen);
2249    struct zink_transfer *trans = (struct zink_transfer *)ptrans;
2250    if (trans->base.b.usage & PIPE_MAP_ONCE && !trans->staging_res)
2251       do_transfer_unmap(screen, trans);
2252    transfer_unmap(pctx, ptrans);
2253 }
2254
2255 static void
2256 zink_image_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
2257 {
2258    struct zink_screen *screen = zink_screen(pctx->screen);
2259    struct zink_transfer *trans = (struct zink_transfer *)ptrans;
2260    if (sizeof(void*) == 4)
2261       do_transfer_unmap(screen, trans);
2262    transfer_unmap(pctx, ptrans);
2263 }
2264
2265 static void
2266 zink_buffer_subdata(struct pipe_context *ctx, struct pipe_resource *buffer,
2267                     unsigned usage, unsigned offset, unsigned size, const void *data)
2268 {
2269    struct pipe_transfer *transfer = NULL;
2270    struct pipe_box box;
2271    uint8_t *map = NULL;
2272
2273    usage |= PIPE_MAP_WRITE;
2274
2275    if (!(usage & PIPE_MAP_DIRECTLY))
2276       usage |= PIPE_MAP_DISCARD_RANGE;
2277
2278    u_box_1d(offset, size, &box);
2279    map = zink_buffer_map(ctx, buffer, 0, usage, &box, &transfer);
2280    if (!map)
2281       return;
2282
2283    memcpy(map, data, size);
2284    zink_buffer_unmap(ctx, transfer);
2285 }
2286
2287 static struct pipe_resource *
2288 zink_resource_get_separate_stencil(struct pipe_resource *pres)
2289 {
2290    /* For packed depth-stencil, we treat depth as the primary resource
2291     * and store S8 as the "second plane" resource.
2292     */
2293    if (pres->next && pres->next->format == PIPE_FORMAT_S8_UINT)
2294       return pres->next;
2295
2296    return NULL;
2297
2298 }
2299
2300 static bool
2301 resource_object_add_bind(struct zink_context *ctx, struct zink_resource *res, unsigned bind)
2302 {
2303    /* base resource already has the cap */
2304    if (res->base.b.bind & bind)
2305       return true;
2306    if (res->obj->is_buffer) {
2307       unreachable("zink: all buffers should have this bit");
2308       return true;
2309    }
2310    assert(!res->obj->dt);
2311    zink_fb_clears_apply_region(ctx, &res->base.b, (struct u_rect){0, res->base.b.width0, 0, res->base.b.height0});
2312    bool ret = add_resource_bind(ctx, res, bind);
2313    if (ret)
2314       zink_resource_rebind(ctx, res);
2315
2316    return ret;
2317 }
2318
2319 bool
2320 zink_resource_object_init_storage(struct zink_context *ctx, struct zink_resource *res)
2321 {
2322    return resource_object_add_bind(ctx, res, PIPE_BIND_SHADER_IMAGE);
2323 }
2324
2325 bool
2326 zink_resource_object_init_mutable(struct zink_context *ctx, struct zink_resource *res)
2327 {
2328    return resource_object_add_bind(ctx, res, ZINK_BIND_MUTABLE);
2329 }
2330
2331 VkDeviceAddress
2332 zink_resource_get_address(struct zink_screen *screen, struct zink_resource *res)
2333 {
2334    assert(res->obj->is_buffer);
2335    if (!res->obj->bda) {
2336       VkBufferDeviceAddressInfo info = {
2337          VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO,
2338          NULL,
2339          res->obj->buffer
2340       };
2341       res->obj->bda = VKSCR(GetBufferDeviceAddress)(screen->dev, &info);
2342    }
2343    return res->obj->bda;
2344 }
2345
2346 void
2347 zink_resource_setup_transfer_layouts(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst)
2348 {
2349    if (src == dst) {
2350       /* The Vulkan 1.1 specification says the following about valid usage
2351        * of vkCmdBlitImage:
2352        *
2353        * "srcImageLayout must be VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR,
2354        *  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL or VK_IMAGE_LAYOUT_GENERAL"
2355        *
2356        * and:
2357        *
2358        * "dstImageLayout must be VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR,
2359        *  VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL or VK_IMAGE_LAYOUT_GENERAL"
2360        *
2361        * Since we cant have the same image in two states at the same time,
2362        * we're effectively left with VK_IMAGE_LAYOUT_SHARED_PRESENT_KHR or
2363        * VK_IMAGE_LAYOUT_GENERAL. And since this isn't a present-related
2364        * operation, VK_IMAGE_LAYOUT_GENERAL seems most appropriate.
2365        */
2366       zink_screen(ctx->base.screen)->image_barrier(ctx, src,
2367                                   VK_IMAGE_LAYOUT_GENERAL,
2368                                   VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
2369                                   VK_PIPELINE_STAGE_TRANSFER_BIT);
2370    } else {
2371       zink_screen(ctx->base.screen)->image_barrier(ctx, src,
2372                                   VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
2373                                   VK_ACCESS_TRANSFER_READ_BIT,
2374                                   VK_PIPELINE_STAGE_TRANSFER_BIT);
2375
2376       zink_screen(ctx->base.screen)->image_barrier(ctx, dst,
2377                                   VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
2378                                   VK_ACCESS_TRANSFER_WRITE_BIT,
2379                                   VK_PIPELINE_STAGE_TRANSFER_BIT);
2380    }
2381 }
2382
2383 void
2384 zink_get_depth_stencil_resources(struct pipe_resource *res,
2385                                  struct zink_resource **out_z,
2386                                  struct zink_resource **out_s)
2387 {
2388    if (!res) {
2389       if (out_z) *out_z = NULL;
2390       if (out_s) *out_s = NULL;
2391       return;
2392    }
2393
2394    if (res->format != PIPE_FORMAT_S8_UINT) {
2395       if (out_z) *out_z = zink_resource(res);
2396       if (out_s) *out_s = zink_resource(zink_resource_get_separate_stencil(res));
2397    } else {
2398       if (out_z) *out_z = NULL;
2399       if (out_s) *out_s = zink_resource(res);
2400    }
2401 }
2402
2403 static void
2404 zink_resource_set_separate_stencil(struct pipe_resource *pres,
2405                                    struct pipe_resource *stencil)
2406 {
2407    assert(util_format_has_depth(util_format_description(pres->format)));
2408    pipe_resource_reference(&pres->next, stencil);
2409 }
2410
2411 static enum pipe_format
2412 zink_resource_get_internal_format(struct pipe_resource *pres)
2413 {
2414    struct zink_resource *res = zink_resource(pres);
2415    return res->internal_format;
2416 }
2417
2418 static const struct u_transfer_vtbl transfer_vtbl = {
2419    .resource_create       = zink_resource_create,
2420    .resource_destroy      = zink_resource_destroy,
2421    .transfer_map          = zink_image_map,
2422    .transfer_unmap        = zink_image_unmap,
2423    .transfer_flush_region = zink_transfer_flush_region,
2424    .get_internal_format   = zink_resource_get_internal_format,
2425    .set_stencil           = zink_resource_set_separate_stencil,
2426    .get_stencil           = zink_resource_get_separate_stencil,
2427 };
2428
2429 bool
2430 zink_screen_resource_init(struct pipe_screen *pscreen)
2431 {
2432    struct zink_screen *screen = zink_screen(pscreen);
2433    pscreen->resource_create = u_transfer_helper_resource_create;
2434    pscreen->resource_create_with_modifiers = zink_resource_create_with_modifiers;
2435    pscreen->resource_create_drawable = zink_resource_create_drawable;
2436    pscreen->resource_destroy = u_transfer_helper_resource_destroy;
2437    pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
2438       U_TRANSFER_HELPER_SEPARATE_Z32S8 | U_TRANSFER_HELPER_SEPARATE_STENCIL |
2439       U_TRANSFER_HELPER_INTERLEAVE_IN_PLACE |
2440       U_TRANSFER_HELPER_MSAA_MAP |
2441       (!screen->have_D24_UNORM_S8_UINT ? U_TRANSFER_HELPER_Z24_IN_Z32F : 0));
2442
2443    if (screen->info.have_KHR_external_memory_fd || screen->info.have_KHR_external_memory_win32) {
2444       pscreen->resource_get_handle = zink_resource_get_handle;
2445       pscreen->resource_from_handle = zink_resource_from_handle;
2446    }
2447    if (screen->instance_info.have_KHR_external_memory_capabilities) {
2448       pscreen->memobj_create_from_handle = zink_memobj_create_from_handle;
2449       pscreen->memobj_destroy = zink_memobj_destroy;
2450       pscreen->resource_from_memobj = zink_resource_from_memobj;
2451    }
2452    pscreen->resource_get_param = zink_resource_get_param;
2453    return true;
2454 }
2455
2456 void
2457 zink_context_resource_init(struct pipe_context *pctx)
2458 {
2459    pctx->buffer_map = zink_buffer_map;
2460    pctx->buffer_unmap = zink_buffer_unmap;
2461    pctx->texture_map = u_transfer_helper_transfer_map;
2462    pctx->texture_unmap = u_transfer_helper_transfer_unmap;
2463
2464    pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
2465    pctx->buffer_subdata = zink_buffer_subdata;
2466    pctx->texture_subdata = u_default_texture_subdata;
2467    pctx->invalidate_resource = zink_resource_invalidate;
2468 }