src/gallium/drivers/d3d12/d3d12_resource.cpp

   1 /*
   2  * Copyright © Microsoft Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "d3d12_resource.h"
  25
  26 #include "d3d12_blit.h"
  27 #include "d3d12_context.h"
  28 #include "d3d12_format.h"
  29 #include "d3d12_screen.h"
  30 #include "d3d12_debug.h"
  31
  32 #include "pipebuffer/pb_bufmgr.h"
  33 #include "util/slab.h"
  34 #include "util/format/u_format.h"
  35 #include "util/u_inlines.h"
  36 #include "util/u_memory.h"
  37 #include "util/format/u_format_zs.h"
  38
  39 #include "frontend/sw_winsys.h"
  40
  41 #include <directx/d3d12.h>
  42 #include <dxguids/dxguids.h>
  43 #include <memory>
  44
  45 #ifndef GENERIC_ALL
  46  // This is only added to winadapter.h in newer DirectX-Headers
  47 #define GENERIC_ALL 0x10000000L
  48 #endif
  49
  50 static bool
  51 can_map_directly(struct pipe_resource *pres)
  52 {
  53    return pres->target == PIPE_BUFFER &&
  54           pres->usage != PIPE_USAGE_DEFAULT &&
  55           pres->usage != PIPE_USAGE_IMMUTABLE;
  56 }
  57
  58 static void
  59 init_valid_range(struct d3d12_resource *res)
  60 {
  61    if (can_map_directly(&res->base.b))
  62       util_range_init(&res->valid_buffer_range);
  63 }
  64
  65 static void
  66 d3d12_resource_destroy(struct pipe_screen *pscreen,
  67                        struct pipe_resource *presource)
  68 {
  69    struct d3d12_resource *resource = d3d12_resource(presource);
  70    threaded_resource_deinit(presource);
  71    if (can_map_directly(presource))
  72       util_range_destroy(&resource->valid_buffer_range);
  73    if (resource->bo)
  74       d3d12_bo_unreference(resource->bo);
  75    FREE(resource);
  76 }
  77
  78 static bool
  79 resource_is_busy(struct d3d12_context *ctx,
  80                  struct d3d12_resource *res,
  81                  bool want_to_write)
  82 {
  83    if (d3d12_batch_has_references(d3d12_current_batch(ctx), res->bo, want_to_write))
  84       return true;
  85
  86    bool busy = false;
  87    d3d12_foreach_submitted_batch(ctx, batch) {
  88       if (!d3d12_reset_batch(ctx, batch, 0))
  89          busy |= d3d12_batch_has_references(batch, res->bo, want_to_write);
  90    }
  91    return busy;
  92 }
  93
  94 void
  95 d3d12_resource_wait_idle(struct d3d12_context *ctx,
  96                          struct d3d12_resource *res,
  97                          bool want_to_write)
  98 {
  99    if (d3d12_batch_has_references(d3d12_current_batch(ctx), res->bo, want_to_write)) {
 100       d3d12_flush_cmdlist_and_wait(ctx);
 101    } else {
 102       d3d12_foreach_submitted_batch(ctx, batch) {
 103          if (d3d12_batch_has_references(batch, res->bo, want_to_write))
 104             d3d12_reset_batch(ctx, batch, PIPE_TIMEOUT_INFINITE);
 105       }
 106    }
 107 }
 108
 109 void
 110 d3d12_resource_release(struct d3d12_resource *resource)
 111 {
 112    if (!resource->bo)
 113       return;
 114    d3d12_bo_unreference(resource->bo);
 115    resource->bo = NULL;
 116 }
 117
 118 static bool
 119 init_buffer(struct d3d12_screen *screen,
 120             struct d3d12_resource *res,
 121             const struct pipe_resource *templ)
 122 {
 123    struct pb_desc buf_desc;
 124    struct pb_manager *bufmgr;
 125    struct pb_buffer *buf;
 126
 127    /* Assert that we don't want to create a buffer with one of the emulated
 128     * formats, these are (currently) only supported when passing the vertex
 129     * element state */
 130    assert(templ->format == d3d12_emulated_vtx_format(templ->format));
 131
 132    switch (templ->usage) {
 133    case PIPE_USAGE_DEFAULT:
 134    case PIPE_USAGE_IMMUTABLE:
 135       bufmgr = screen->cache_bufmgr;
 136       buf_desc.usage = (pb_usage_flags)PB_USAGE_GPU_READ_WRITE;
 137       break;
 138    case PIPE_USAGE_DYNAMIC:
 139    case PIPE_USAGE_STREAM:
 140       bufmgr = screen->slab_bufmgr;
 141       buf_desc.usage = (pb_usage_flags)(PB_USAGE_CPU_WRITE | PB_USAGE_GPU_READ);
 142       break;
 143    case PIPE_USAGE_STAGING:
 144       bufmgr = screen->readback_slab_bufmgr;
 145       buf_desc.usage = (pb_usage_flags)(PB_USAGE_GPU_WRITE | PB_USAGE_CPU_READ_WRITE);
 146       break;
 147    default:
 148       unreachable("Invalid pipe usage");
 149    }
 150
 151    /* We can't suballocate buffers that might be bound as a sampler view, *only*
 152     * because in the case of R32G32B32 formats (12 bytes per pixel), it's not possible
 153     * to guarantee the offset will be divisible.
 154     */
 155    if (templ->bind & PIPE_BIND_SAMPLER_VIEW)
 156       bufmgr = screen->cache_bufmgr;
 157
 158    buf_desc.alignment = D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT;
 159    res->dxgi_format = DXGI_FORMAT_UNKNOWN;
 160    buf = bufmgr->create_buffer(bufmgr, templ->width0, &buf_desc);
 161    if (!buf)
 162       return false;
 163    res->bo = d3d12_bo_wrap_buffer(buf);
 164
 165    return true;
 166 }
 167
 168 static bool
 169 init_texture(struct d3d12_screen *screen,
 170              struct d3d12_resource *res,
 171              const struct pipe_resource *templ)
 172 {
 173    ID3D12Resource *d3d12_res;
 174
 175    res->mip_levels = templ->last_level + 1;
 176    res->dxgi_format = d3d12_get_format(templ->format);
 177
 178    D3D12_RESOURCE_DESC desc;
 179    desc.Format = res->dxgi_format;
 180    desc.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT;
 181    desc.Width = templ->width0;
 182    desc.Height = templ->height0;
 183    desc.DepthOrArraySize = templ->array_size;
 184    desc.MipLevels = templ->last_level + 1;
 185
 186    desc.SampleDesc.Count = MAX2(templ->nr_samples, 1);
 187    desc.SampleDesc.Quality = 0; /* TODO: figure this one out */
 188
 189    switch (templ->target) {
 190    case PIPE_TEXTURE_1D:
 191    case PIPE_TEXTURE_1D_ARRAY:
 192       desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE1D;
 193       break;
 194
 195    case PIPE_TEXTURE_CUBE:
 196    case PIPE_TEXTURE_CUBE_ARRAY:
 197       desc.DepthOrArraySize *= 6;
 198       FALLTHROUGH;
 199    case PIPE_TEXTURE_2D:
 200    case PIPE_TEXTURE_2D_ARRAY:
 201    case PIPE_TEXTURE_RECT:
 202       desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D;
 203       break;
 204
 205    case PIPE_TEXTURE_3D:
 206       desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE3D;
 207       desc.DepthOrArraySize = templ->depth0;
 208       break;
 209
 210    default:
 211       unreachable("Invalid texture type");
 212    }
 213
 214    desc.Flags = D3D12_RESOURCE_FLAG_NONE;
 215
 216    if (templ->bind & PIPE_BIND_SHADER_BUFFER)
 217       desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
 218
 219    if (templ->bind & PIPE_BIND_RENDER_TARGET)
 220       desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET;
 221
 222    if (templ->bind & PIPE_BIND_DEPTH_STENCIL) {
 223       desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL;
 224
 225       /* Sadly, we can't set D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE in the
 226        * case where PIPE_BIND_SAMPLER_VIEW isn't set, because that would
 227        * prevent us from using the resource with u_blitter, which requires
 228        * sneaking in sampler-usage throught the back-door.
 229        */
 230    }
 231
 232    if (screen->support_shader_images && templ->nr_samples <= 1) {
 233       /* Ideally, we'd key off of PIPE_BIND_SHADER_IMAGE for this, but it doesn't
 234        * seem to be set properly. So, all UAV-capable resources need the UAV flag.
 235        */
 236       D3D12_FEATURE_DATA_FORMAT_SUPPORT support = { desc.Format };
 237       if (SUCCEEDED(screen->dev->CheckFeatureSupport(D3D12_FEATURE_FORMAT_SUPPORT, &support, sizeof(support))) &&
 238          (support.Support2 & (D3D12_FORMAT_SUPPORT2_UAV_TYPED_LOAD | D3D12_FORMAT_SUPPORT2_UAV_TYPED_STORE)) ==
 239          (D3D12_FORMAT_SUPPORT2_UAV_TYPED_LOAD | D3D12_FORMAT_SUPPORT2_UAV_TYPED_STORE)) {
 240          desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
 241          desc.Format = d3d12_get_typeless_format(templ->format);
 242       }
 243    }
 244
 245    desc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN;
 246    if (templ->bind & (PIPE_BIND_SCANOUT |
 247                       PIPE_BIND_SHARED | PIPE_BIND_LINEAR))
 248       desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
 249
 250    D3D12_HEAP_PROPERTIES heap_pris = screen->dev->GetCustomHeapProperties(0, D3D12_HEAP_TYPE_DEFAULT);
 251
 252    D3D12_HEAP_FLAGS heap_flags = screen->support_create_not_resident ?
 253       D3D12_HEAP_FLAG_CREATE_NOT_RESIDENT : D3D12_HEAP_FLAG_NONE;
 254    enum d3d12_residency_status init_residency = screen->support_create_not_resident ?
 255       d3d12_evicted : d3d12_resident;
 256
 257    HRESULT hres = screen->dev->CreateCommittedResource(&heap_pris,
 258                                                    heap_flags,
 259                                                    &desc,
 260                                                    D3D12_RESOURCE_STATE_COMMON,
 261                                                    NULL,
 262                                                    IID_PPV_ARGS(&d3d12_res));
 263    if (FAILED(hres))
 264       return false;
 265
 266    if (screen->winsys && (templ->bind & PIPE_BIND_DISPLAY_TARGET)) {
 267       struct sw_winsys *winsys = screen->winsys;
 268       res->dt = winsys->displaytarget_create(screen->winsys,
 269                                              res->base.b.bind,
 270                                              res->base.b.format,
 271                                              templ->width0,
 272                                              templ->height0,
 273                                              64, NULL,
 274                                              &res->dt_stride);
 275    }
 276
 277    res->bo = d3d12_bo_wrap_res(screen, d3d12_res, templ->format, init_residency);
 278
 279    return true;
 280 }
 281
 282 static void
 283 convert_planar_resource(struct d3d12_resource *res)
 284 {
 285    unsigned num_planes = util_format_get_num_planes(res->base.b.format);
 286    if (num_planes <= 1 || res->base.b.next || !res->bo)
 287       return;
 288
 289    struct pipe_resource *next = nullptr;
 290    struct pipe_resource *planes[3] = {
 291       &res->base.b, nullptr, nullptr
 292    };
 293    for (int plane = num_planes - 1; plane >= 0; --plane) {
 294       struct d3d12_resource *plane_res = d3d12_resource(planes[plane]);
 295       if (!plane_res) {
 296          plane_res = CALLOC_STRUCT(d3d12_resource);
 297          *plane_res = *res;
 298          d3d12_bo_reference(plane_res->bo);
 299          pipe_reference_init(&plane_res->base.b.reference, 1);
 300          threaded_resource_init(&plane_res->base.b, false);
 301       }
 302
 303       plane_res->base.b.next = next;
 304       next = &plane_res->base.b;
 305
 306       plane_res->plane_slice = plane;
 307       plane_res->base.b.format = util_format_get_plane_format(res->base.b.format, plane);
 308       plane_res->base.b.width0 = util_format_get_plane_width(res->base.b.format, plane, res->base.b.width0);
 309       plane_res->base.b.height0 = util_format_get_plane_height(res->base.b.format, plane, res->base.b.height0);
 310
 311 #if DEBUG
 312       struct d3d12_screen *screen = d3d12_screen(res->base.b.screen);
 313       D3D12_RESOURCE_DESC desc = res->bo->res->GetDesc();
 314       D3D12_PLACED_SUBRESOURCE_FOOTPRINT placed_footprint = {};
 315       D3D12_SUBRESOURCE_FOOTPRINT *footprint = &placed_footprint.Footprint;
 316       unsigned subresource = plane * desc.MipLevels * desc.DepthOrArraySize;
 317       screen->dev->GetCopyableFootprints(&desc, subresource, 1, 0, &placed_footprint, nullptr, nullptr, nullptr);
 318       assert(plane_res->base.b.width0 == footprint->Width);
 319       assert(plane_res->base.b.height0 == footprint->Height);
 320       assert(plane_res->base.b.depth0 == footprint->Depth);
 321       assert(plane_res->first_plane == &res->base.b);
 322 #endif
 323    }
 324 }
 325
 326 static struct pipe_resource *
 327 d3d12_resource_create(struct pipe_screen *pscreen,
 328                       const struct pipe_resource *templ)
 329 {
 330    struct d3d12_screen *screen = d3d12_screen(pscreen);
 331    struct d3d12_resource *res = CALLOC_STRUCT(d3d12_resource);
 332    bool ret;
 333
 334    res->base.b = *templ;
 335    res->overall_format = templ->format;
 336    res->plane_slice = 0;
 337    res->first_plane = &res->base.b;
 338
 339    if (D3D12_DEBUG_RESOURCE & d3d12_debug) {
 340       debug_printf("D3D12: Create %sresource %s@%d %dx%dx%d as:%d mip:%d\n",
 341                    templ->usage == PIPE_USAGE_STAGING ? "STAGING " :"",
 342                    util_format_name(templ->format), templ->nr_samples,
 343                    templ->width0, templ->height0, templ->depth0,
 344                    templ->array_size, templ->last_level);
 345    }
 346
 347    pipe_reference_init(&res->base.b.reference, 1);
 348    res->base.b.screen = pscreen;
 349
 350    if (templ->target == PIPE_BUFFER) {
 351       ret = init_buffer(screen, res, templ);
 352    } else {
 353       ret = init_texture(screen, res, templ);
 354    }
 355
 356    if (!ret) {
 357       FREE(res);
 358       return NULL;
 359    }
 360
 361    init_valid_range(res);
 362    threaded_resource_init(&res->base.b,
 363       templ->usage == PIPE_USAGE_DEFAULT &&
 364       templ->target == PIPE_BUFFER);
 365
 366    memset(&res->bind_counts, 0, sizeof(d3d12_resource::bind_counts));
 367
 368    convert_planar_resource(res);
 369
 370    return &res->base.b;
 371 }
 372
 373 static struct pipe_resource *
 374 d3d12_resource_from_handle(struct pipe_screen *pscreen,
 375                           const struct pipe_resource *templ,
 376                           struct winsys_handle *handle, unsigned usage)
 377 {
 378    struct d3d12_screen *screen = d3d12_screen(pscreen);
 379    if (handle->type != WINSYS_HANDLE_TYPE_D3D12_RES &&
 380        handle->type != WINSYS_HANDLE_TYPE_FD)
 381       return NULL;
 382
 383    struct d3d12_resource *res = CALLOC_STRUCT(d3d12_resource);
 384    if (!res)
 385       return NULL;
 386
 387    if (templ && templ->next) {
 388       struct d3d12_resource* next = d3d12_resource(templ->next);
 389       if (next->bo) {
 390          res->base.b = *templ;
 391          res->bo = next->bo;
 392          d3d12_bo_reference(res->bo);
 393       }
 394    }
 395
 396    pipe_reference_init(&res->base.b.reference, 1);
 397    res->base.b.screen = pscreen;
 398
 399    ID3D12Resource *d3d12_res = nullptr;
 400    if (res->bo) {
 401       d3d12_res = res->bo->res;
 402    } else if (handle->type == WINSYS_HANDLE_TYPE_D3D12_RES) {
 403       d3d12_res = (ID3D12Resource *)handle->com_obj;
 404    } else {
 405       struct d3d12_screen *screen = d3d12_screen(pscreen);
 406
 407 #ifdef _WIN32
 408       HANDLE d3d_handle = handle->handle;
 409 #else
 410       HANDLE d3d_handle = (HANDLE)(intptr_t)handle->handle;
 411 #endif
 412       screen->dev->OpenSharedHandle(d3d_handle, IID_PPV_ARGS(&d3d12_res));
 413    }
 414
 415    D3D12_PLACED_SUBRESOURCE_FOOTPRINT placed_footprint = {};
 416    D3D12_SUBRESOURCE_FOOTPRINT *footprint = &placed_footprint.Footprint;
 417    D3D12_RESOURCE_DESC incoming_res_desc;
 418
 419    if (!d3d12_res)
 420       goto invalid;
 421
 422    incoming_res_desc = d3d12_res->GetDesc();
 423
 424    /* Get a description for this plane */
 425    if (templ && handle->format != templ->format) {
 426       unsigned subresource = handle->plane * incoming_res_desc.MipLevels * incoming_res_desc.DepthOrArraySize;
 427       screen->dev->GetCopyableFootprints(&incoming_res_desc, subresource, 1, 0, &placed_footprint, nullptr, nullptr, nullptr);
 428    } else {
 429       footprint->Format = incoming_res_desc.Format;
 430       footprint->Width = incoming_res_desc.Width;
 431       footprint->Height = incoming_res_desc.Height;
 432       footprint->Depth = incoming_res_desc.DepthOrArraySize;
 433    }
 434
 435    if (footprint->Width > UINT32_MAX ||
 436        footprint->Height > UINT16_MAX) {
 437       debug_printf("d3d12: Importing resource too large\n");
 438       goto invalid;
 439    }
 440    res->base.b.width0 = incoming_res_desc.Width;
 441    res->base.b.height0 = incoming_res_desc.Height;
 442    res->base.b.depth0 = 1;
 443    res->base.b.array_size = 1;
 444
 445    switch (incoming_res_desc.Dimension) {
 446    case D3D12_RESOURCE_DIMENSION_BUFFER:
 447       res->base.b.target = PIPE_BUFFER;
 448       res->base.b.bind = PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_CONSTANT_BUFFER |
 449          PIPE_BIND_INDEX_BUFFER | PIPE_BIND_STREAM_OUTPUT | PIPE_BIND_SHADER_BUFFER |
 450          PIPE_BIND_COMMAND_ARGS_BUFFER | PIPE_BIND_QUERY_BUFFER;
 451       break;
 452    case D3D12_RESOURCE_DIMENSION_TEXTURE1D:
 453       res->base.b.target = incoming_res_desc.DepthOrArraySize > 1 ?
 454          PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
 455       res->base.b.array_size = incoming_res_desc.DepthOrArraySize;
 456       break;
 457    case D3D12_RESOURCE_DIMENSION_TEXTURE2D:
 458       res->base.b.target = incoming_res_desc.DepthOrArraySize > 1 ?
 459          PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
 460       res->base.b.array_size = incoming_res_desc.DepthOrArraySize;
 461       break;
 462    case D3D12_RESOURCE_DIMENSION_TEXTURE3D:
 463       res->base.b.target = PIPE_TEXTURE_3D;
 464       res->base.b.depth0 = footprint->Depth;
 465       break;
 466    default:
 467       unreachable("Invalid dimension");
 468       break;
 469    }
 470    res->base.b.nr_samples = incoming_res_desc.SampleDesc.Count;
 471    res->base.b.last_level = incoming_res_desc.MipLevels - 1;
 472    res->base.b.usage = PIPE_USAGE_DEFAULT;
 473    if (incoming_res_desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)
 474       res->base.b.bind |= PIPE_BIND_RENDER_TARGET | PIPE_BIND_BLENDABLE | PIPE_BIND_DISPLAY_TARGET;
 475    if (incoming_res_desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)
 476       res->base.b.bind |= PIPE_BIND_DEPTH_STENCIL;
 477    if (incoming_res_desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS)
 478       res->base.b.bind |= PIPE_BIND_SHADER_IMAGE;
 479    if ((incoming_res_desc.Flags & D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE) == D3D12_RESOURCE_FLAG_NONE)
 480       res->base.b.bind |= PIPE_BIND_SAMPLER_VIEW;
 481
 482    if (templ) {
 483       if (res->base.b.target == PIPE_TEXTURE_2D_ARRAY &&
 484             (templ->target == PIPE_TEXTURE_CUBE ||
 485              templ->target == PIPE_TEXTURE_CUBE_ARRAY)) {
 486          if (res->base.b.array_size < 6) {
 487             debug_printf("d3d12: Importing cube resource with too few array layers\n");
 488             goto invalid;
 489          }
 490          res->base.b.target = templ->target;
 491          res->base.b.array_size /= 6;
 492       }
 493       unsigned templ_samples = MAX2(templ->nr_samples, 1);
 494       if (res->base.b.target != templ->target ||
 495           footprint->Width != templ->width0 ||
 496           footprint->Height != templ->height0 ||
 497           footprint->Depth != templ->depth0 ||
 498           res->base.b.array_size != templ->array_size ||
 499           incoming_res_desc.SampleDesc.Count != templ_samples ||
 500           res->base.b.last_level != templ->last_level) {
 501          debug_printf("d3d12: Importing resource with mismatched dimensions: "
 502             "plane: %d, target: %d vs %d, width: %d vs %d, height: %d vs %d, "
 503             "depth: %d vs %d, array_size: %d vs %d, samples: %d vs %d, mips: %d vs %d\n",
 504             handle->plane,
 505             res->base.b.target, templ->target,
 506             footprint->Width, templ->width0,
 507             footprint->Height, templ->height0,
 508             footprint->Depth, templ->depth0,
 509             res->base.b.array_size, templ->array_size,
 510             incoming_res_desc.SampleDesc.Count, templ_samples,
 511             res->base.b.last_level + 1, templ->last_level + 1);
 512          goto invalid;
 513       }
 514       if ((footprint->Format != d3d12_get_format(templ->format) &&
 515            footprint->Format != d3d12_get_typeless_format(templ->format)) ||
 516           (incoming_res_desc.Format != d3d12_get_format((enum pipe_format)handle->format) &&
 517            incoming_res_desc.Format != d3d12_get_typeless_format((enum pipe_format)handle->format))) {
 518          debug_printf("d3d12: Importing resource with mismatched format: "
 519             "plane could be DXGI format %d or %d, but is %d, "
 520             "overall could be DXGI format %d or %d, but is %d\n",
 521             d3d12_get_format(templ->format),
 522             d3d12_get_typeless_format(templ->format),
 523             footprint->Format,
 524             d3d12_get_format((enum pipe_format)handle->format),
 525             d3d12_get_typeless_format((enum pipe_format)handle->format),
 526             incoming_res_desc.Format);
 527          goto invalid;
 528       }
 529       if (templ->bind & ~res->base.b.bind) {
 530          debug_printf("d3d12: Imported resource doesn't have necessary bind flags\n");
 531          goto invalid;
 532       }
 533
 534       res->base.b.format = templ->format;
 535       res->overall_format = (enum pipe_format)handle->format;
 536    } else {
 537       /* Search the pipe format lookup table for an entry */
 538       res->base.b.format = d3d12_get_pipe_format(incoming_res_desc.Format);
 539
 540       if (res->base.b.format == PIPE_FORMAT_NONE) {
 541          /* Convert from typeless to a reasonable default */
 542          res->base.b.format = d3d12_get_default_pipe_format(incoming_res_desc.Format);
 543
 544          if (res->base.b.format == PIPE_FORMAT_NONE) {
 545             debug_printf("d3d12: Unable to deduce non-typeless resource format %d\n", incoming_res_desc.Format);
 546             goto invalid;
 547          }
 548       }
 549
 550       res->overall_format = res->base.b.format;
 551    }
 552
 553    if (!templ)
 554       handle->format = res->overall_format;
 555
 556    res->dxgi_format = d3d12_get_format(res->overall_format);
 557    res->plane_slice = handle->plane;
 558    res->first_plane = &res->base.b;
 559
 560    if (!res->bo) {
 561       res->bo = d3d12_bo_wrap_res(screen, d3d12_res, res->overall_format, d3d12_permanently_resident);
 562    }
 563    init_valid_range(res);
 564
 565    threaded_resource_init(&res->base.b, false);
 566    convert_planar_resource(res);
 567
 568    return &res->base.b;
 569
 570 invalid:
 571    if (res->bo)
 572       d3d12_bo_unreference(res->bo);
 573    else if (d3d12_res)
 574       d3d12_res->Release();
 575    FREE(res);
 576    return NULL;
 577 }
 578
 579 static bool
 580 d3d12_resource_get_handle(struct pipe_screen *pscreen,
 581                           struct pipe_context *pcontext,
 582                           struct pipe_resource *pres,
 583                           struct winsys_handle *handle,
 584                           unsigned usage)
 585 {
 586    struct d3d12_resource *res = d3d12_resource(pres);
 587    struct d3d12_screen *screen = d3d12_screen(pscreen);
 588
 589    switch (handle->type) {
 590    case WINSYS_HANDLE_TYPE_D3D12_RES:
 591       handle->com_obj = d3d12_resource_resource(res);
 592       return true;
 593    case WINSYS_HANDLE_TYPE_FD: {
 594       HANDLE d3d_handle = nullptr;
 595
 596       screen->dev->CreateSharedHandle(d3d12_resource_resource(res),
 597                                       nullptr,
 598                                       GENERIC_ALL,
 599                                       nullptr,
 600                                       &d3d_handle);
 601       if (!d3d_handle)
 602          return false;
 603
 604 #ifdef _WIN32
 605       handle->handle = d3d_handle;
 606 #else
 607       handle->handle = (int)(intptr_t)d3d_handle;
 608 #endif
 609       handle->format = pres->format;
 610       handle->modifier = ~0ull;
 611       return true;
 612    }
 613    default:
 614       return false;
 615    }
 616 }
 617
 618 struct pipe_resource *
 619 d3d12_resource_from_resource(struct pipe_screen *pscreen,
 620                               ID3D12Resource* input_res)
 621 {
 622     D3D12_RESOURCE_DESC input_desc = input_res->GetDesc();
 623     struct winsys_handle handle;
 624     memset(&handle, 0, sizeof(handle));
 625     handle.type = WINSYS_HANDLE_TYPE_D3D12_RES;
 626     handle.format = d3d12_get_pipe_format(input_desc.Format);
 627     handle.com_obj = input_res;
 628
 629     struct pipe_resource templ;
 630     memset(&templ, 0, sizeof(templ));
 631     if(input_desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) {
 632        templ.target = PIPE_BUFFER;
 633     } else {
 634       templ.target = (input_desc.DepthOrArraySize > 1) ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
 635     }
 636
 637     templ.format = d3d12_get_pipe_format(input_desc.Format);
 638     templ.width0 = input_desc.Width;
 639     templ.height0 = input_desc.Height;
 640     templ.depth0 = input_desc.DepthOrArraySize;
 641     templ.array_size = input_desc.DepthOrArraySize;
 642     templ.flags = 0;
 643
 644     return d3d12_resource_from_handle(
 645         pscreen,
 646         &templ,
 647         &handle,
 648         PIPE_USAGE_DEFAULT
 649     );
 650 }
 651
 652 /**
 653  * On Map/Unmap operations, we readback or flush all the underlying planes
 654  * of planar resources. The map/unmap operation from the caller is
 655  * expected to be done for res->plane_slice plane only, but some
 656  * callers expect adjacent allocations for next contiguous plane access
 657  *
 658  * In this function, we take the res and box the caller passed, and the plane_* properties
 659  * that are currently being readback/flushed, and adjust the d3d12_transfer ptrans
 660  * accordingly for the GPU copy operation between planes.
 661  */
 662 static void d3d12_adjust_transfer_dimensions_for_plane(const struct d3d12_resource *res,
 663                                                        unsigned plane_slice,
 664                                                        unsigned plane_stride,
 665                                                        unsigned plane_layer_stride,
 666                                                        unsigned plane_offset,
 667                                                        const struct pipe_box* original_box,
 668                                                        struct pipe_transfer *ptrans/*inout*/)
 669 {
 670    /* Adjust strides, offsets to the corresponding plane*/
 671    ptrans->stride = plane_stride;
 672    ptrans->layer_stride = plane_layer_stride;
 673    ptrans->offset = plane_offset;
 674
 675    /* Find multipliers such that:*/
 676    /* first_plane.width = width_multiplier * planes[res->plane_slice].width*/
 677    /* first_plane.height = height_multiplier * planes[res->plane_slice].height*/
 678    float width_multiplier = res->first_plane->width0 / (float) util_format_get_plane_width(res->overall_format, res->plane_slice, res->first_plane->width0);
 679    float height_multiplier = res->first_plane->height0 / (float) util_format_get_plane_height(res->overall_format, res->plane_slice, res->first_plane->height0);
 680
 681    /* Normalize box back to overall dimensions (first plane)*/
 682    ptrans->box.width = width_multiplier * original_box->width;
 683    ptrans->box.height = height_multiplier * original_box->height;
 684    ptrans->box.x = width_multiplier * original_box->x;
 685    ptrans->box.y = height_multiplier * original_box->y;
 686
 687    /* Now adjust dimensions to plane_slice*/
 688    ptrans->box.width = util_format_get_plane_width(res->overall_format, plane_slice, ptrans->box.width);
 689    ptrans->box.height = util_format_get_plane_height(res->overall_format, plane_slice, ptrans->box.height);
 690    ptrans->box.x = util_format_get_plane_width(res->overall_format, plane_slice, ptrans->box.x);
 691    ptrans->box.y = util_format_get_plane_height(res->overall_format, plane_slice, ptrans->box.y);
 692 }
 693
 694 static
 695 void d3d12_resource_get_planes_info(pipe_resource *pres,
 696                                     unsigned num_planes,
 697                                     pipe_resource **planes,
 698                                     unsigned *strides,
 699                                     unsigned *layer_strides,
 700                                     unsigned *offsets,
 701                                     unsigned *staging_res_size)
 702 {
 703    struct d3d12_resource* res = d3d12_resource(pres);
 704    *staging_res_size = 0;
 705    struct pipe_resource *cur_plane_resource = res->first_plane;
 706    for (uint plane_slice = 0; plane_slice < num_planes; ++plane_slice) {
 707       planes[plane_slice] = cur_plane_resource;
 708       int width = util_format_get_plane_width(res->base.b.format, plane_slice, res->first_plane->width0);
 709       int height = util_format_get_plane_height(res->base.b.format, plane_slice, res->first_plane->height0);
 710
 711       strides[plane_slice] = align(util_format_get_stride(cur_plane_resource->format, width),
 712                            D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
 713
 714       layer_strides[plane_slice] = align(util_format_get_2d_size(cur_plane_resource->format,
 715                                                    strides[plane_slice],
 716                                                    height),
 717                                  D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT);
 718
 719       offsets[plane_slice] = *staging_res_size;
 720       *staging_res_size += layer_strides[plane_slice];
 721       cur_plane_resource = cur_plane_resource->next;
 722    }
 723 }
 724
 725 static constexpr unsigned d3d12_max_planes = 3;
 726
 727 /**
 728  * Get stride and offset for the given pipe resource without the need to get
 729  * a winsys_handle.
 730  */
 731 void
 732 d3d12_resource_get_info(struct pipe_screen *pscreen,
 733                         struct pipe_resource *pres,
 734                         unsigned *stride,
 735                         unsigned *offset)
 736 {
 737
 738    struct d3d12_resource* res = d3d12_resource(pres);
 739    unsigned num_planes = util_format_get_num_planes(res->overall_format);
 740
 741    pipe_resource *planes[d3d12_max_planes];
 742    unsigned int strides[d3d12_max_planes];
 743    unsigned int layer_strides[d3d12_max_planes];
 744    unsigned int offsets[d3d12_max_planes];
 745    unsigned staging_res_size = 0;
 746    d3d12_resource_get_planes_info(
 747       pres,
 748       num_planes,
 749       planes,
 750       strides,
 751       layer_strides,
 752       offsets,
 753       &staging_res_size
 754    );
 755
 756    if(stride) {
 757       *stride = strides[res->plane_slice];
 758    }
 759
 760    if(offset) {
 761       *offset = offsets[res->plane_slice];
 762    }
 763 }
 764
 765 void
 766 d3d12_screen_resource_init(struct pipe_screen *pscreen)
 767 {
 768    pscreen->resource_create = d3d12_resource_create;
 769    pscreen->resource_from_handle = d3d12_resource_from_handle;
 770    pscreen->resource_get_handle = d3d12_resource_get_handle;
 771    pscreen->resource_destroy = d3d12_resource_destroy;
 772    pscreen->resource_get_info = d3d12_resource_get_info;
 773 }
 774
 775 unsigned int
 776 get_subresource_id(struct d3d12_resource *res, unsigned resid,
 777                    unsigned z, unsigned base_level)
 778 {
 779    unsigned resource_stride = res->base.b.last_level + 1;
 780    if (res->base.b.target == PIPE_TEXTURE_1D_ARRAY ||
 781        res->base.b.target == PIPE_TEXTURE_2D_ARRAY)
 782       resource_stride *= res->base.b.array_size;
 783
 784    if (res->base.b.target == PIPE_TEXTURE_CUBE)
 785       resource_stride *= 6;
 786
 787    if (res->base.b.target == PIPE_TEXTURE_CUBE_ARRAY)
 788       resource_stride *= 6 * res->base.b.array_size;
 789
 790    unsigned layer_stride = res->base.b.last_level + 1;
 791
 792    return resid * resource_stride + z * layer_stride +
 793          base_level + res->plane_slice * resource_stride;
 794 }
 795
 796 static D3D12_TEXTURE_COPY_LOCATION
 797 fill_texture_location(struct d3d12_resource *res,
 798                       struct d3d12_transfer *trans, unsigned resid, unsigned z)
 799 {
 800    D3D12_TEXTURE_COPY_LOCATION tex_loc = {0};
 801    int subres = get_subresource_id(res, resid, z, trans->base.b.level);
 802
 803    tex_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
 804    tex_loc.SubresourceIndex = subres;
 805    tex_loc.pResource = d3d12_resource_resource(res);
 806    return tex_loc;
 807 }
 808
 809 static D3D12_TEXTURE_COPY_LOCATION
 810 fill_buffer_location(struct d3d12_context *ctx,
 811                      struct d3d12_resource *res,
 812                      struct d3d12_resource *staging_res,
 813                      struct d3d12_transfer *trans,
 814                      unsigned depth,
 815                      unsigned resid, unsigned z)
 816 {
 817    D3D12_TEXTURE_COPY_LOCATION buf_loc = {0};
 818    D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint;
 819    uint64_t offset = 0;
 820    auto descr = d3d12_resource_underlying(res, &offset)->GetDesc();
 821    struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
 822    ID3D12Device* dev = screen->dev;
 823
 824    unsigned sub_resid = get_subresource_id(res, resid, z, trans->base.b.level);
 825    dev->GetCopyableFootprints(&descr, sub_resid, 1, 0, &footprint, nullptr, nullptr, nullptr);
 826
 827    buf_loc.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
 828    buf_loc.pResource = d3d12_resource_underlying(staging_res, &offset);
 829    buf_loc.PlacedFootprint = footprint;
 830    buf_loc.PlacedFootprint.Offset += offset;
 831    buf_loc.PlacedFootprint.Offset += trans->base.b.offset;
 832
 833    if (util_format_has_depth(util_format_description(res->base.b.format)) &&
 834        screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
 835       buf_loc.PlacedFootprint.Footprint.Width = res->base.b.width0;
 836       buf_loc.PlacedFootprint.Footprint.Height = res->base.b.height0;
 837       buf_loc.PlacedFootprint.Footprint.Depth = res->base.b.depth0;
 838    } else {
 839       buf_loc.PlacedFootprint.Footprint.Width = ALIGN(trans->base.b.box.width,
 840                                                       util_format_get_blockwidth(res->base.b.format));
 841       buf_loc.PlacedFootprint.Footprint.Height = ALIGN(trans->base.b.box.height,
 842                                                        util_format_get_blockheight(res->base.b.format));
 843       buf_loc.PlacedFootprint.Footprint.Depth = ALIGN(depth,
 844                                                       util_format_get_blockdepth(res->base.b.format));
 845    }
 846
 847    buf_loc.PlacedFootprint.Footprint.RowPitch = trans->base.b.stride;
 848
 849    return buf_loc;
 850 }
 851
 852 struct copy_info {
 853    struct d3d12_resource *dst;
 854    D3D12_TEXTURE_COPY_LOCATION dst_loc;
 855    UINT dst_x, dst_y, dst_z;
 856    struct d3d12_resource *src;
 857    D3D12_TEXTURE_COPY_LOCATION src_loc;
 858    D3D12_BOX *src_box;
 859 };
 860
 861
 862 static void
 863 copy_texture_region(struct d3d12_context *ctx,
 864                     struct copy_info& info)
 865 {
 866    auto batch = d3d12_current_batch(ctx);
 867
 868    d3d12_batch_reference_resource(batch, info.src, false);
 869    d3d12_batch_reference_resource(batch, info.dst, true);
 870    d3d12_transition_resource_state(ctx, info.src, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_BIND_INVALIDATE_FULL);
 871    d3d12_transition_resource_state(ctx, info.dst, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_BIND_INVALIDATE_FULL);
 872    d3d12_apply_resource_states(ctx, false);
 873    ctx->cmdlist->CopyTextureRegion(&info.dst_loc, info.dst_x, info.dst_y, info.dst_z,
 874                                    &info.src_loc, info.src_box);
 875 }
 876
 877 static void
 878 transfer_buf_to_image_part(struct d3d12_context *ctx,
 879                            struct d3d12_resource *res,
 880                            struct d3d12_resource *staging_res,
 881                            struct d3d12_transfer *trans,
 882                            int z, int depth, int start_z, int dest_z,
 883                            int resid)
 884 {
 885    if (D3D12_DEBUG_RESOURCE & d3d12_debug) {
 886       debug_printf("D3D12: Copy %dx%dx%d + %dx%dx%d from buffer %s to image %s\n",
 887                    trans->base.b.box.x, trans->base.b.box.y, trans->base.b.box.z,
 888                    trans->base.b.box.width, trans->base.b.box.height, trans->base.b.box.depth,
 889                    util_format_name(staging_res->base.b.format),
 890                    util_format_name(res->base.b.format));
 891    }
 892
 893    struct d3d12_screen *screen = d3d12_screen(res->base.b.screen);
 894    struct copy_info copy_info;
 895    copy_info.src = staging_res;
 896    copy_info.src_loc = fill_buffer_location(ctx, res, staging_res, trans, depth, resid, z);
 897    copy_info.src_loc.PlacedFootprint.Offset += (z  - start_z) * trans->base.b.layer_stride;
 898    copy_info.src_box = nullptr;
 899    copy_info.dst = res;
 900    copy_info.dst_loc = fill_texture_location(res, trans, resid, z);
 901    if (util_format_has_depth(util_format_description(res->base.b.format)) &&
 902        screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
 903       copy_info.dst_x = 0;
 904       copy_info.dst_y = 0;
 905    } else {
 906       copy_info.dst_x = trans->base.b.box.x;
 907       copy_info.dst_y = trans->base.b.box.y;
 908    }
 909    copy_info.dst_z = res->base.b.target == PIPE_TEXTURE_CUBE ? 0 : dest_z;
 910    copy_info.src_box = nullptr;
 911
 912    copy_texture_region(ctx, copy_info);
 913 }
 914
 915 static bool
 916 transfer_buf_to_image(struct d3d12_context *ctx,
 917                       struct d3d12_resource *res,
 918                       struct d3d12_resource *staging_res,
 919                       struct d3d12_transfer *trans, int resid)
 920 {
 921    if (res->base.b.target == PIPE_TEXTURE_3D) {
 922       assert(resid == 0);
 923       transfer_buf_to_image_part(ctx, res, staging_res, trans,
 924                                  0, trans->base.b.box.depth, 0,
 925                                  trans->base.b.box.z, 0);
 926    } else {
 927       int num_layers = trans->base.b.box.depth;
 928       int start_z = trans->base.b.box.z;
 929
 930       for (int z = start_z; z < start_z + num_layers; ++z) {
 931          transfer_buf_to_image_part(ctx, res, staging_res, trans,
 932                                            z, 1, start_z, 0, resid);
 933       }
 934    }
 935    return true;
 936 }
 937
 938 static void
 939 transfer_image_part_to_buf(struct d3d12_context *ctx,
 940                            struct d3d12_resource *res,
 941                            struct d3d12_resource *staging_res,
 942                            struct d3d12_transfer *trans,
 943                            unsigned resid, int z, int start_layer,
 944                            int start_box_z, int depth)
 945 {
 946    struct pipe_box *box = &trans->base.b.box;
 947    D3D12_BOX src_box = {};
 948
 949    struct d3d12_screen *screen = d3d12_screen(res->base.b.screen);
 950    struct copy_info copy_info;
 951    copy_info.src_box = nullptr;
 952    copy_info.src = res;
 953    copy_info.src_loc = fill_texture_location(res, trans, resid, z);
 954    copy_info.dst = staging_res;
 955    copy_info.dst_loc = fill_buffer_location(ctx, res, staging_res, trans,
 956                                             depth, resid, z);
 957    copy_info.dst_loc.PlacedFootprint.Offset += (z  - start_layer) * trans->base.b.layer_stride;
 958    copy_info.dst_x = copy_info.dst_y = copy_info.dst_z = 0;
 959
 960    bool whole_resource = util_texrange_covers_whole_level(&res->base.b, trans->base.b.level,
 961                                                           box->x, box->y, start_box_z,
 962                                                           box->width, box->height, depth);
 963    if (util_format_has_depth(util_format_description(res->base.b.format)) &&
 964        screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED)
 965       whole_resource = true;
 966    if (!whole_resource) {
 967       src_box.left = box->x;
 968       src_box.right = box->x + box->width;
 969       src_box.top = box->y;
 970       src_box.bottom = box->y + box->height;
 971       src_box.front = start_box_z;
 972       src_box.back = start_box_z + depth;
 973       copy_info.src_box = &src_box;
 974    }
 975
 976    copy_texture_region(ctx, copy_info);
 977 }
 978
 979 static bool
 980 transfer_image_to_buf(struct d3d12_context *ctx,
 981                             struct d3d12_resource *res,
 982                             struct d3d12_resource *staging_res,
 983                             struct d3d12_transfer *trans,
 984                             unsigned resid)
 985 {
 986
 987    /* We only suppport loading from either an texture array
 988     * or a ZS texture, so either resid is zero, or num_layers == 1)
 989     */
 990    assert(resid == 0 || trans->base.b.box.depth == 1);
 991
 992    if (D3D12_DEBUG_RESOURCE & d3d12_debug) {
 993       debug_printf("D3D12: Copy %dx%dx%d + %dx%dx%d from %s@%d to %s\n",
 994                    trans->base.b.box.x, trans->base.b.box.y, trans->base.b.box.z,
 995                    trans->base.b.box.width, trans->base.b.box.height, trans->base.b.box.depth,
 996                    util_format_name(res->base.b.format), resid,
 997                    util_format_name(staging_res->base.b.format));
 998    }
 999
1000    struct pipe_resource *resolved_resource = nullptr;
1001    if (res->base.b.nr_samples > 1) {
1002       struct pipe_resource tmpl = res->base.b;
1003       tmpl.nr_samples = 0;
1004       resolved_resource = d3d12_resource_create(ctx->base.screen, &tmpl);
1005       struct pipe_blit_info resolve_info = {};
1006       struct pipe_box box = {0,0,0, (int)res->base.b.width0, (int16_t)res->base.b.height0, (int16_t)res->base.b.depth0};
1007       resolve_info.dst.resource = resolved_resource;
1008       resolve_info.dst.box = box;
1009       resolve_info.dst.format = res->base.b.format;
1010       resolve_info.src.resource = &res->base.b;
1011       resolve_info.src.box = box;
1012       resolve_info.src.format = res->base.b.format;
1013       resolve_info.filter = PIPE_TEX_FILTER_NEAREST;
1014       resolve_info.mask = util_format_get_mask(tmpl.format);
1015
1016
1017
1018       d3d12_blit(&ctx->base, &resolve_info);
1019       res = (struct d3d12_resource *)resolved_resource;
1020    }
1021
1022
1023    if (res->base.b.target == PIPE_TEXTURE_3D) {
1024       transfer_image_part_to_buf(ctx, res, staging_res, trans, resid,
1025                                  0, 0, trans->base.b.box.z, trans->base.b.box.depth);
1026    } else {
1027       int start_layer = trans->base.b.box.z;
1028       for (int z = start_layer; z < start_layer + trans->base.b.box.depth; ++z) {
1029          transfer_image_part_to_buf(ctx, res, staging_res, trans, resid,
1030                                     z, start_layer, 0, 1);
1031       }
1032    }
1033
1034    pipe_resource_reference(&resolved_resource, NULL);
1035
1036    return true;
1037 }
1038
1039 static void
1040 transfer_buf_to_buf(struct d3d12_context *ctx,
1041                     struct d3d12_resource *src,
1042                     struct d3d12_resource *dst,
1043                     uint64_t src_offset,
1044                     uint64_t dst_offset,
1045                     uint64_t width)
1046 {
1047    auto batch = d3d12_current_batch(ctx);
1048
1049    d3d12_batch_reference_resource(batch, src, false);
1050    d3d12_batch_reference_resource(batch, dst, true);
1051
1052    uint64_t src_offset_suballoc = 0;
1053    uint64_t dst_offset_suballoc = 0;
1054    auto src_d3d12 = d3d12_resource_underlying(src, &src_offset_suballoc);
1055    auto dst_d3d12 = d3d12_resource_underlying(dst, &dst_offset_suballoc);
1056    src_offset += src_offset_suballoc;
1057    dst_offset += dst_offset_suballoc;
1058
1059    // Same-resource copies not supported, since the resource would need to be in both states
1060    assert(src_d3d12 != dst_d3d12);
1061    d3d12_transition_resource_state(ctx, src, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_BIND_INVALIDATE_FULL);
1062    d3d12_transition_resource_state(ctx, dst, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_BIND_INVALIDATE_FULL);
1063    d3d12_apply_resource_states(ctx, false);
1064    ctx->cmdlist->CopyBufferRegion(dst_d3d12, dst_offset,
1065                                   src_d3d12, src_offset,
1066                                   width);
1067 }
1068
1069 static unsigned
1070 linear_offset(int x, int y, int z, unsigned stride, unsigned layer_stride)
1071 {
1072    return x +
1073           y * stride +
1074           z * layer_stride;
1075 }
1076
1077 static D3D12_RANGE
1078 linear_range(const struct pipe_box *box, unsigned stride, unsigned layer_stride)
1079 {
1080    D3D12_RANGE range;
1081
1082    range.Begin = linear_offset(box->x, box->y, box->z,
1083                                stride, layer_stride);
1084    range.End = linear_offset(box->x + box->width,
1085                              box->y + box->height - 1,
1086                              box->z + box->depth - 1,
1087                              stride, layer_stride);
1088
1089    return range;
1090 }
1091
1092 static bool
1093 synchronize(struct d3d12_context *ctx,
1094             struct d3d12_resource *res,
1095             unsigned usage,
1096             D3D12_RANGE *range)
1097 {
1098    assert(can_map_directly(&res->base.b));
1099
1100    /* Check whether that range contains valid data; if not, we might not need to sync */
1101    if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
1102        usage & PIPE_MAP_WRITE &&
1103        !util_ranges_intersect(&res->valid_buffer_range, range->Begin, range->End)) {
1104       usage |= PIPE_MAP_UNSYNCHRONIZED;
1105    }
1106
1107    if (!(usage & PIPE_MAP_UNSYNCHRONIZED) && resource_is_busy(ctx, res, usage & PIPE_MAP_WRITE)) {
1108       if (usage & PIPE_MAP_DONTBLOCK) {
1109          if (d3d12_batch_has_references(d3d12_current_batch(ctx), res->bo, usage & PIPE_MAP_WRITE))
1110             d3d12_flush_cmdlist(ctx);
1111          return false;
1112       }
1113
1114       d3d12_resource_wait_idle(ctx, res, usage & PIPE_MAP_WRITE);
1115    }
1116
1117    if (usage & PIPE_MAP_WRITE)
1118       util_range_add(&res->base.b, &res->valid_buffer_range,
1119                      range->Begin, range->End);
1120
1121    return true;
1122 }
1123
1124 /* A wrapper to make sure local resources are freed and unmapped with
1125  * any exit path */
1126 struct local_resource {
1127    local_resource(pipe_screen *s, struct pipe_resource *tmpl) :
1128       mapped(false)
1129    {
1130       res = d3d12_resource(d3d12_resource_create(s, tmpl));
1131    }
1132
1133    ~local_resource() {
1134       if (res) {
1135          if (mapped)
1136             d3d12_bo_unmap(res->bo, nullptr);
1137          pipe_resource_reference((struct pipe_resource **)&res, NULL);
1138       }
1139    }
1140
1141    void *
1142    map() {
1143       void *ptr;
1144       ptr = d3d12_bo_map(res->bo, nullptr);
1145       if (ptr)
1146          mapped = true;
1147       return ptr;
1148    }
1149
1150    void unmap()
1151    {
1152       if (mapped)
1153          d3d12_bo_unmap(res->bo, nullptr);
1154       mapped = false;
1155    }
1156
1157    operator struct d3d12_resource *() {
1158       return res;
1159    }
1160
1161    bool operator !() {
1162       return !res;
1163    }
1164 private:
1165    struct d3d12_resource *res;
1166    bool mapped;
1167 };
1168
1169 /* Combined depth-stencil needs a special handling for reading back: DX handled
1170  * depth and stencil parts as separate resources and handles copying them only
1171  * by using seperate texture copy calls with different formats. So create two
1172  * buffers, read back both resources and interleave the data.
1173  */
1174 static void
1175 prepare_zs_layer_strides(struct d3d12_screen *screen,
1176                          struct d3d12_resource *res,
1177                          const struct pipe_box *box,
1178                          struct d3d12_transfer *trans)
1179 {
1180    bool copy_whole_resource = screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED;
1181    int width = copy_whole_resource ? res->base.b.width0 : box->width;
1182    int height = copy_whole_resource ? res->base.b.height0 : box->height;
1183
1184    trans->base.b.stride = align(util_format_get_stride(res->base.b.format, width),
1185                                 D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
1186    trans->base.b.layer_stride = util_format_get_2d_size(res->base.b.format,
1187                                                         trans->base.b.stride,
1188                                                         height);
1189
1190    if (copy_whole_resource) {
1191       trans->zs_cpu_copy_stride = align(util_format_get_stride(res->base.b.format, box->width),
1192                                         D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
1193       trans->zs_cpu_copy_layer_stride = util_format_get_2d_size(res->base.b.format,
1194                                                                 trans->base.b.stride,
1195                                                                 box->height);
1196    } else {
1197       trans->zs_cpu_copy_stride = trans->base.b.stride;
1198       trans->zs_cpu_copy_layer_stride = trans->base.b.layer_stride;
1199    }
1200 }
1201
1202 static void *
1203 read_zs_surface(struct d3d12_context *ctx, struct d3d12_resource *res,
1204                 const struct pipe_box *box,
1205                 struct d3d12_transfer *trans)
1206 {
1207    pipe_screen *pscreen = ctx->base.screen;
1208    struct d3d12_screen *screen = d3d12_screen(pscreen);
1209
1210    prepare_zs_layer_strides(screen, res, box, trans);
1211
1212    struct pipe_resource tmpl;
1213    memset(&tmpl, 0, sizeof tmpl);
1214    tmpl.target = PIPE_BUFFER;
1215    tmpl.format = PIPE_FORMAT_R32_UNORM;
1216    tmpl.bind = 0;
1217    tmpl.usage = PIPE_USAGE_STAGING;
1218    tmpl.flags = 0;
1219    tmpl.width0 = trans->base.b.layer_stride;
1220    tmpl.height0 = 1;
1221    tmpl.depth0 = 1;
1222    tmpl.array_size = 1;
1223
1224    local_resource depth_buffer(pscreen, &tmpl);
1225    if (!depth_buffer) {
1226       debug_printf("Allocating staging buffer for depth failed\n");
1227       return NULL;
1228    }
1229
1230    if (!transfer_image_to_buf(ctx, res, depth_buffer, trans, 0))
1231       return NULL;
1232
1233    tmpl.format = PIPE_FORMAT_R8_UINT;
1234
1235    local_resource stencil_buffer(pscreen, &tmpl);
1236    if (!stencil_buffer) {
1237       debug_printf("Allocating staging buffer for stencilfailed\n");
1238       return NULL;
1239    }
1240
1241    if (!transfer_image_to_buf(ctx, res, stencil_buffer, trans, 1))
1242       return NULL;
1243
1244    d3d12_flush_cmdlist_and_wait(ctx);
1245
1246    uint8_t *depth_ptr = (uint8_t *)depth_buffer.map();
1247    if (!depth_ptr) {
1248       debug_printf("Mapping staging depth buffer failed\n");
1249       return NULL;
1250    }
1251
1252    uint8_t *stencil_ptr =  (uint8_t *)stencil_buffer.map();
1253    if (!stencil_ptr) {
1254       debug_printf("Mapping staging stencil buffer failed\n");
1255       return NULL;
1256    }
1257
1258    uint8_t *buf = (uint8_t *)malloc(trans->zs_cpu_copy_layer_stride);
1259    if (!buf)
1260       return NULL;
1261
1262    trans->data = buf;
1263
1264    switch (res->base.b.format) {
1265    case PIPE_FORMAT_Z24_UNORM_S8_UINT:
1266       if (screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
1267          depth_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1268          stencil_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1269       }
1270       util_format_z24_unorm_s8_uint_pack_separate(buf, trans->zs_cpu_copy_stride,
1271                                                   (uint32_t *)depth_ptr, trans->base.b.stride,
1272                                                   stencil_ptr, trans->base.b.stride,
1273                                                   trans->base.b.box.width, trans->base.b.box.height);
1274       break;
1275    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1276       if (screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
1277          depth_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1278          stencil_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x;
1279       }
1280       util_format_z32_float_s8x24_uint_pack_z_float(buf, trans->zs_cpu_copy_stride,
1281                                                     (float *)depth_ptr, trans->base.b.stride,
1282                                                     trans->base.b.box.width, trans->base.b.box.height);
1283       util_format_z32_float_s8x24_uint_pack_s_8uint(buf, trans->zs_cpu_copy_stride,
1284                                                     stencil_ptr, trans->base.b.stride,
1285                                                     trans->base.b.box.width, trans->base.b.box.height);
1286       break;
1287    default:
1288       unreachable("Unsupported depth steancil format");
1289    };
1290
1291    return trans->data;
1292 }
1293
1294 static void *
1295 prepare_write_zs_surface(struct d3d12_resource *res,
1296                          const struct pipe_box *box,
1297                          struct d3d12_transfer *trans)
1298 {
1299    struct d3d12_screen *screen = d3d12_screen(res->base.b.screen);
1300    prepare_zs_layer_strides(screen, res, box, trans);
1301    uint32_t *buf = (uint32_t *)malloc(trans->base.b.layer_stride);
1302    if (!buf)
1303       return NULL;
1304
1305    trans->data = buf;
1306    return trans->data;
1307 }
1308
1309 static void
1310 write_zs_surface(struct pipe_context *pctx, struct d3d12_resource *res,
1311                  struct d3d12_transfer *trans)
1312 {
1313    struct d3d12_screen *screen = d3d12_screen(res->base.b.screen);
1314    struct pipe_resource tmpl;
1315    memset(&tmpl, 0, sizeof tmpl);
1316    tmpl.target = PIPE_BUFFER;
1317    tmpl.format = PIPE_FORMAT_R32_UNORM;
1318    tmpl.bind = 0;
1319    tmpl.usage = PIPE_USAGE_STAGING;
1320    tmpl.flags = 0;
1321    tmpl.width0 = trans->base.b.layer_stride;
1322    tmpl.height0 = 1;
1323    tmpl.depth0 = 1;
1324    tmpl.array_size = 1;
1325
1326    local_resource depth_buffer(pctx->screen, &tmpl);
1327    if (!depth_buffer) {
1328       debug_printf("Allocating staging buffer for depth failed\n");
1329       return;
1330    }
1331
1332    local_resource stencil_buffer(pctx->screen, &tmpl);
1333    if (!stencil_buffer) {
1334       debug_printf("Allocating staging buffer for depth failed\n");
1335       return;
1336    }
1337
1338    uint8_t *depth_ptr = (uint8_t *)depth_buffer.map();
1339    if (!depth_ptr) {
1340       debug_printf("Mapping staging depth buffer failed\n");
1341       return;
1342    }
1343
1344    uint8_t *stencil_ptr =  (uint8_t *)stencil_buffer.map();
1345    if (!stencil_ptr) {
1346       debug_printf("Mapping staging stencil buffer failed\n");
1347       return;
1348    }
1349
1350    switch (res->base.b.format) {
1351    case PIPE_FORMAT_Z24_UNORM_S8_UINT:
1352       if (screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
1353          depth_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1354          stencil_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1355       }
1356       util_format_z32_unorm_unpack_z_32unorm((uint32_t *)depth_ptr, trans->base.b.stride, (uint8_t*)trans->data,
1357                                              trans->zs_cpu_copy_stride, trans->base.b.box.width,
1358                                              trans->base.b.box.height);
1359       util_format_z24_unorm_s8_uint_unpack_s_8uint(stencil_ptr, trans->base.b.stride, (uint8_t*)trans->data,
1360                                                    trans->zs_cpu_copy_stride, trans->base.b.box.width,
1361                                                    trans->base.b.box.height);
1362       break;
1363    case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1364       if (screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
1365          depth_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x * 4;
1366          stencil_ptr += trans->base.b.box.y * trans->base.b.stride + trans->base.b.box.x;
1367       }
1368       util_format_z32_float_s8x24_uint_unpack_z_float((float *)depth_ptr, trans->base.b.stride, (uint8_t*)trans->data,
1369                                                       trans->zs_cpu_copy_stride, trans->base.b.box.width,
1370                                                       trans->base.b.box.height);
1371       util_format_z32_float_s8x24_uint_unpack_s_8uint(stencil_ptr, trans->base.b.stride, (uint8_t*)trans->data,
1372                                                       trans->zs_cpu_copy_stride, trans->base.b.box.width,
1373                                                       trans->base.b.box.height);
1374       break;
1375    default:
1376       unreachable("Unsupported depth steancil format");
1377    };
1378
1379    stencil_buffer.unmap();
1380    depth_buffer.unmap();
1381
1382    transfer_buf_to_image(d3d12_context(pctx), res, depth_buffer, trans, 0);
1383    transfer_buf_to_image(d3d12_context(pctx), res, stencil_buffer, trans, 1);
1384 }
1385
1386 #define BUFFER_MAP_ALIGNMENT 64
1387
1388 static void *
1389 d3d12_transfer_map(struct pipe_context *pctx,
1390                    struct pipe_resource *pres,
1391                    unsigned level,
1392                    unsigned usage,
1393                    const struct pipe_box *box,
1394                    struct pipe_transfer **transfer)
1395 {
1396    struct d3d12_context *ctx = d3d12_context(pctx);
1397    struct d3d12_resource *res = d3d12_resource(pres);
1398    struct d3d12_screen *screen = d3d12_screen(pres->screen);
1399
1400    if (usage & PIPE_MAP_DIRECTLY || !res->bo)
1401       return NULL;
1402
1403    slab_child_pool* transfer_pool = (usage & TC_TRANSFER_MAP_THREADED_UNSYNC) ?
1404       &ctx->transfer_pool_unsync : &ctx->transfer_pool;
1405    struct d3d12_transfer *trans = (struct d3d12_transfer *)slab_zalloc(transfer_pool);
1406    struct pipe_transfer *ptrans = &trans->base.b;
1407    if (!trans)
1408       return NULL;
1409
1410    ptrans->level = level;
1411    ptrans->usage = (enum pipe_map_flags)usage;
1412    ptrans->box = *box;
1413
1414    D3D12_RANGE range;
1415    range.Begin = 0;
1416
1417    void *ptr;
1418    if (can_map_directly(&res->base.b)) {
1419       if (pres->target == PIPE_BUFFER) {
1420          ptrans->stride = 0;
1421          ptrans->layer_stride = 0;
1422       } else {
1423          ptrans->stride = util_format_get_stride(pres->format, box->width);
1424          ptrans->layer_stride = util_format_get_2d_size(pres->format,
1425                                                         ptrans->stride,
1426                                                         box->height);
1427       }
1428
1429       range = linear_range(box, ptrans->stride, ptrans->layer_stride);
1430       if (!synchronize(ctx, res, usage, &range)) {
1431          slab_free(transfer_pool, trans);
1432          return NULL;
1433       }
1434       ptr = d3d12_bo_map(res->bo, &range);
1435    } else if (unlikely(pres->format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
1436                        pres->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)) {
1437       if (usage & PIPE_MAP_READ) {
1438          ptr = read_zs_surface(ctx, res, box, trans);
1439       } else if (usage & PIPE_MAP_WRITE){
1440          ptr = prepare_write_zs_surface(res, box, trans);
1441       } else {
1442          ptr = nullptr;
1443       }
1444    } else if(util_format_is_yuv(res->overall_format)) {
1445
1446       /* Get planes information*/
1447
1448       unsigned num_planes = util_format_get_num_planes(res->overall_format);
1449       pipe_resource *planes[d3d12_max_planes];
1450       unsigned int strides[d3d12_max_planes];
1451       unsigned int layer_strides[d3d12_max_planes];
1452       unsigned int offsets[d3d12_max_planes];
1453       unsigned staging_res_size = 0;
1454
1455       d3d12_resource_get_planes_info(
1456          pres,
1457          num_planes,
1458          planes,
1459          strides,
1460          layer_strides,
1461          offsets,
1462          &staging_res_size
1463       );
1464
1465       /* Allocate a buffer for all the planes to fit in adjacent memory*/
1466
1467       pipe_resource_usage staging_usage = (usage & (PIPE_MAP_READ | PIPE_MAP_READ_WRITE)) ?
1468          PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
1469       trans->staging_res = pipe_buffer_create(pctx->screen, 0,
1470                                               staging_usage,
1471                                               staging_res_size);
1472       if (!trans->staging_res)
1473          return NULL;
1474
1475       struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res);
1476
1477       /* Readback contents into the buffer allocation now if map was intended for read*/
1478
1479       /* Read all planes if readback needed*/
1480       if (usage & PIPE_MAP_READ) {
1481          pipe_box original_box = ptrans->box;
1482          for (uint plane_slice = 0; plane_slice < num_planes; ++plane_slice) {
1483             /* Adjust strides, offsets, box to the corresponding plane for the copytexture operation*/
1484             d3d12_adjust_transfer_dimensions_for_plane(res,
1485                                                        plane_slice,
1486                                                        strides[plane_slice],
1487                                                        layer_strides[plane_slice],
1488                                                        offsets[plane_slice],
1489                                                        &original_box,
1490                                                        ptrans/*inout*/);
1491             /* Perform the readback*/
1492             if(!transfer_image_to_buf(ctx, d3d12_resource(planes[plane_slice]), staging_res, trans, 0)){
1493                return NULL;
1494             }
1495          }
1496
1497          d3d12_flush_cmdlist_and_wait(ctx);
1498       }
1499
1500       /* Map the whole staging buffer containing all the planes contiguously*/
1501       /* Just offset the resulting ptr to the according plane offset*/
1502
1503       range.End = staging_res_size - range.Begin;
1504       uint8_t* all_planes_map = (uint8_t*) d3d12_bo_map(staging_res->bo, &range);
1505
1506       ptrans->stride = strides[res->plane_slice];
1507       ptrans->layer_stride = layer_strides[res->plane_slice];
1508       ptr = all_planes_map + offsets[res->plane_slice];
1509
1510    } else {
1511       ptrans->stride = align(util_format_get_stride(pres->format, box->width),
1512                               D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
1513       ptrans->layer_stride = util_format_get_2d_size(pres->format,
1514                                                      ptrans->stride,
1515                                                      box->height);
1516
1517       if (res->base.b.target != PIPE_TEXTURE_3D)
1518          ptrans->layer_stride = align(ptrans->layer_stride,
1519                                       D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT);
1520
1521       if (util_format_has_depth(util_format_description(pres->format)) &&
1522           screen->opts2.ProgrammableSamplePositionsTier == D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED) {
1523          trans->zs_cpu_copy_stride = ptrans->stride;
1524          trans->zs_cpu_copy_layer_stride = ptrans->layer_stride;
1525
1526          ptrans->stride = align(util_format_get_stride(pres->format, pres->width0),
1527                                 D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
1528          ptrans->layer_stride = util_format_get_2d_size(pres->format,
1529                                                         ptrans->stride,
1530                                                         pres->height0);
1531
1532          range.Begin = box->y * ptrans->stride +
1533             box->x * util_format_get_blocksize(pres->format);
1534       }
1535
1536       unsigned staging_res_size = ptrans->layer_stride * box->depth;
1537       if (res->base.b.target == PIPE_BUFFER) {
1538          /* To properly support ARB_map_buffer_alignment, we need to return a pointer
1539           * that's appropriately offset from a 64-byte-aligned base address.
1540           */
1541          assert(box->x >= 0);
1542          unsigned aligned_x = (unsigned)box->x % BUFFER_MAP_ALIGNMENT;
1543          staging_res_size = align(box->width + aligned_x,
1544                                   D3D12_TEXTURE_DATA_PITCH_ALIGNMENT);
1545          range.Begin = aligned_x;
1546       }
1547
1548       pipe_resource_usage staging_usage = (usage & (PIPE_MAP_DISCARD_RANGE | PIPE_MAP_DISCARD_WHOLE_RESOURCE)) ?
1549          PIPE_USAGE_STREAM : PIPE_USAGE_STAGING;
1550
1551       trans->staging_res = pipe_buffer_create(pctx->screen, 0,
1552                                               staging_usage,
1553                                               staging_res_size);
1554       if (!trans->staging_res) {
1555          slab_free(transfer_pool, trans);
1556          return NULL;
1557       }
1558
1559       struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res);
1560
1561       if ((usage & (PIPE_MAP_DISCARD_RANGE | PIPE_MAP_DISCARD_WHOLE_RESOURCE | TC_TRANSFER_MAP_THREADED_UNSYNC)) == 0) {
1562          bool ret = true;
1563          if (pres->target == PIPE_BUFFER) {
1564             uint64_t src_offset = box->x;
1565             uint64_t dst_offset = src_offset % BUFFER_MAP_ALIGNMENT;
1566             transfer_buf_to_buf(ctx, res, staging_res, src_offset, dst_offset, box->width);
1567          } else
1568             ret = transfer_image_to_buf(ctx, res, staging_res, trans, 0);
1569          if (!ret)
1570             return NULL;
1571          d3d12_flush_cmdlist_and_wait(ctx);
1572       }
1573
1574       range.End = staging_res_size - range.Begin;
1575
1576       ptr = d3d12_bo_map(staging_res->bo, &range);
1577    }
1578
1579    pipe_resource_reference(&ptrans->resource, pres);
1580    *transfer = ptrans;
1581    return ptr;
1582 }
1583
1584 static void
1585 d3d12_transfer_unmap(struct pipe_context *pctx,
1586                      struct pipe_transfer *ptrans)
1587 {
1588    struct d3d12_context *ctx = d3d12_context(pctx);
1589    struct d3d12_resource *res = d3d12_resource(ptrans->resource);
1590    struct d3d12_transfer *trans = (struct d3d12_transfer *)ptrans;
1591    D3D12_RANGE range = { 0, 0 };
1592
1593    if (trans->data != nullptr) {
1594       if (trans->base.b.usage & PIPE_MAP_WRITE)
1595          write_zs_surface(pctx, res, trans);
1596       free(trans->data);
1597    } else if (trans->staging_res) {
1598       if(util_format_is_yuv(res->overall_format)) {
1599
1600          /* Get planes information*/
1601          unsigned num_planes = util_format_get_num_planes(res->overall_format);
1602          pipe_resource *planes[d3d12_max_planes];
1603          unsigned int strides[d3d12_max_planes];
1604          unsigned int layer_strides[d3d12_max_planes];
1605          unsigned int offsets[d3d12_max_planes];
1606          unsigned staging_res_size = 0;
1607
1608          d3d12_resource_get_planes_info(
1609             ptrans->resource,
1610             num_planes,
1611             planes,
1612             strides,
1613             layer_strides,
1614             offsets,
1615             &staging_res_size
1616          );
1617
1618          /* Flush the changed contents into the GPU texture*/
1619
1620          /* In theory we should just flush only the contents for the plane*/
1621          /* requested in res->plane_slice, but the VAAPI frontend has this*/
1622          /* behaviour in which they assume that mapping the first plane of*/
1623          /* NV12, P010, etc resources will will give them a buffer containing*/
1624          /* both Y and UV planes contigously in vaDeriveImage and then vaMapBuffer*/
1625          /* so, flush them all*/
1626
1627          struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res);
1628          if (trans->base.b.usage & PIPE_MAP_WRITE) {
1629             assert(ptrans->box.x >= 0);
1630             range.Begin = res->base.b.target == PIPE_BUFFER ?
1631                (unsigned)ptrans->box.x % BUFFER_MAP_ALIGNMENT : 0;
1632             range.End = staging_res->base.b.width0 - range.Begin;
1633
1634             d3d12_bo_unmap(staging_res->bo, &range);
1635             pipe_box original_box = ptrans->box;
1636             for (uint plane_slice = 0; plane_slice < num_planes; ++plane_slice) {
1637                /* Adjust strides, offsets to the corresponding plane for the copytexture operation*/
1638                d3d12_adjust_transfer_dimensions_for_plane(res,
1639                                                           plane_slice,
1640                                                           strides[plane_slice],
1641                                                           layer_strides[plane_slice],
1642                                                           offsets[plane_slice],
1643                                                           &original_box,
1644                                                           ptrans/*inout*/);
1645
1646                transfer_buf_to_image(ctx, d3d12_resource(planes[plane_slice]), staging_res, trans, 0);
1647             }
1648          }
1649
1650          pipe_resource_reference(&trans->staging_res, NULL);
1651       } else {
1652          struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res);
1653          if (trans->base.b.usage & PIPE_MAP_WRITE) {
1654             assert(ptrans->box.x >= 0);
1655             range.Begin = res->base.b.target == PIPE_BUFFER ?
1656                (unsigned)ptrans->box.x % BUFFER_MAP_ALIGNMENT : 0;
1657             range.End = staging_res->base.b.width0 - range.Begin;
1658          }
1659          d3d12_bo_unmap(staging_res->bo, &range);
1660
1661          if (trans->base.b.usage & PIPE_MAP_WRITE) {
1662             struct d3d12_context *ctx = d3d12_context(pctx);
1663             if (res->base.b.target == PIPE_BUFFER) {
1664                uint64_t dst_offset = trans->base.b.box.x;
1665                uint64_t src_offset = dst_offset % BUFFER_MAP_ALIGNMENT;
1666                transfer_buf_to_buf(ctx, staging_res, res, src_offset, dst_offset, ptrans->box.width);
1667             } else
1668                transfer_buf_to_image(ctx, res, staging_res, trans, 0);
1669          }
1670
1671          pipe_resource_reference(&trans->staging_res, NULL);
1672       }
1673    } else {
1674       if (trans->base.b.usage & PIPE_MAP_WRITE) {
1675          range.Begin = ptrans->box.x;
1676          range.End = ptrans->box.x + ptrans->box.width;
1677       }
1678       d3d12_bo_unmap(res->bo, &range);
1679    }
1680
1681    pipe_resource_reference(&ptrans->resource, NULL);
1682    slab_free(&d3d12_context(pctx)->transfer_pool, ptrans);
1683 }
1684
1685 void
1686 d3d12_context_resource_init(struct pipe_context *pctx)
1687 {
1688    pctx->buffer_map = d3d12_transfer_map;
1689    pctx->buffer_unmap = d3d12_transfer_unmap;
1690    pctx->texture_map = d3d12_transfer_map;
1691    pctx->texture_unmap = d3d12_transfer_unmap;
1692
1693    pctx->transfer_flush_region = u_default_transfer_flush_region;
1694    pctx->buffer_subdata = u_default_buffer_subdata;
1695    pctx->texture_subdata = u_default_texture_subdata;
1696 }