ext/vulkan/vkfullscreenrender.c

   1 /*
   2  * GStreamer
   3  * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Library General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Library General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Library General Public
  16  * License along with this library; if not, write to the
  17  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
  18  * Boston, MA 02110-1301, USA.
  19  */
  20
  21 /**
  22  * SECTION:element-vulkanimageidentity
  23  * @title: vulkanimgeidentity
  24  *
  25  * vulkanimageidentity produces a vulkan image that is a copy of the input image.
  26  */
  27
  28 #ifdef HAVE_CONFIG_H
  29 #include "config.h"
  30 #endif
  31
  32 #include <string.h>
  33
  34 #include "vkimageidentity.h"
  35 #include "vkshader.h"
  36
  37 GST_DEBUG_CATEGORY (gst_debug_vulkan_full_screen_render);
  38 #define GST_CAT_DEFAULT gst_debug_vulkan_full_screen_render
  39
  40 struct Vertex vertices[] = {
  41   {-1.0f, -1.0f, 0.0f, 0.0f, 0.0f},
  42   {1.0f, -1.0f, 0.0f, 1.0f, 0.0f},
  43   {1.0f, 1.0f, 0.0f, 1.0f, 1.0f},
  44   {-1.0f, 1.0f, 0.0f, 0.0f, 1.0f},
  45 };
  46
  47 gushort indices[] = {
  48   0, 1, 2, 0, 2, 3,
  49 };
  50
  51 static void gst_vulkan_full_screen_render_finalize (GObject * object);
  52
  53 static gboolean gst_vulkan_full_screen_render_query (GstBaseTransform * bt,
  54     GstPadDirection direction, GstQuery * query);
  55 static void gst_vulkan_full_screen_render_set_context (GstElement * element,
  56     GstContext * context);
  57
  58 static gboolean gst_vulkan_full_screen_render_start (GstBaseTransform * bt);
  59 static gboolean gst_vulkan_full_screen_render_stop (GstBaseTransform * bt);
  60
  61 static gboolean gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt,
  62     GstCaps * in_caps, GstCaps * out_caps);
  63 static GstCaps *gst_vulkan_full_screen_render_transform_caps (GstBaseTransform *
  64     bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
  65 static gboolean
  66 gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt,
  67     GstQuery * decide_query, GstQuery * query);
  68 static gboolean
  69 gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt,
  70     GstQuery * query);
  71
  72 #define IMAGE_FORMATS " { BGRA }"
  73
  74 static GstStaticPadTemplate gst_vulkan_sink_template =
  75 GST_STATIC_PAD_TEMPLATE ("sink",
  76     GST_PAD_SINK,
  77     GST_PAD_ALWAYS,
  78     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  79         (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
  80             IMAGE_FORMATS)));
  81
  82 static GstStaticPadTemplate gst_vulkan_src_template =
  83 GST_STATIC_PAD_TEMPLATE ("src",
  84     GST_PAD_SRC,
  85     GST_PAD_ALWAYS,
  86     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  87         (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
  88             IMAGE_FORMATS)));
  89
  90 enum
  91 {
  92   PROP_0,
  93 };
  94
  95 enum
  96 {
  97   SIGNAL_0,
  98   LAST_SIGNAL
  99 };
 100
 101 /* static guint gst_vulkan_full_screen_render_signals[LAST_SIGNAL] = { 0 }; */
 102
 103 #define gst_vulkan_full_screen_render_parent_class parent_class
 104 G_DEFINE_TYPE_WITH_CODE (GstVulkanFullScreenRender,
 105     gst_vulkan_full_screen_render, GST_TYPE_BASE_TRANSFORM,
 106     GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_full_screen_render,
 107         "vulkanimageidentity", 0, "Vulkan Image identity"));
 108
 109 static void
 110 gst_vulkan_full_screen_render_class_init (GstVulkanFullScreenRenderClass *
 111     klass)
 112 {
 113   GObjectClass *gobject_class;
 114   GstElementClass *gstelement_class;
 115   GstBaseTransformClass *gstbasetransform_class;
 116
 117   gobject_class = (GObjectClass *) klass;
 118   gstelement_class = (GstElementClass *) klass;
 119   gstbasetransform_class = (GstBaseTransformClass *) klass;
 120
 121   gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader",
 122       "Filter/Video", "A Vulkan image copier",
 123       "Matthew Waters <matthew@centricular.com>");
 124
 125   gst_element_class_add_static_pad_template (gstelement_class,
 126       &gst_vulkan_sink_template);
 127   gst_element_class_add_static_pad_template (gstelement_class,
 128       &gst_vulkan_src_template);
 129
 130   gobject_class->finalize = gst_vulkan_full_screen_render_finalize;
 131
 132   gstelement_class->set_context = gst_vulkan_full_screen_render_set_context;
 133   gstbasetransform_class->start =
 134       GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_start);
 135   gstbasetransform_class->stop =
 136       GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_stop);
 137   gstbasetransform_class->query =
 138       GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_query);
 139   gstbasetransform_class->set_caps = gst_vulkan_full_screen_render_set_caps;
 140   gstbasetransform_class->transform_caps =
 141       gst_vulkan_full_screen_render_transform_caps;
 142   gstbasetransform_class->propose_allocation =
 143       gst_vulkan_full_screen_render_propose_allocation;
 144   gstbasetransform_class->decide_allocation =
 145       gst_vulkan_full_screen_render_decide_allocation;
 146 }
 147
 148 static void
 149 gst_vulkan_full_screen_render_init (GstVulkanFullScreenRender * render)
 150 {
 151 }
 152
 153 static void
 154 gst_vulkan_full_screen_render_finalize (GObject * object)
 155 {
 156   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (object);
 157
 158   gst_caps_replace (&render->in_caps, NULL);
 159   gst_caps_replace (&render->out_caps, NULL);
 160
 161   G_OBJECT_CLASS (parent_class)->finalize (object);
 162 }
 163
 164 static gboolean
 165 gst_vulkan_full_screen_render_query (GstBaseTransform * bt,
 166     GstPadDirection direction, GstQuery * query)
 167 {
 168   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
 169
 170   switch (GST_QUERY_TYPE (query)) {
 171     case GST_QUERY_CONTEXT:{
 172       if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query,
 173               NULL, render->instance, render->device))
 174         return TRUE;
 175
 176       if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render),
 177               query, render->queue))
 178         return TRUE;
 179
 180       break;
 181     }
 182     default:
 183       break;
 184   }
 185
 186   return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query);
 187 }
 188
 189 static void
 190 gst_vulkan_full_screen_render_set_context (GstElement * element,
 191     GstContext * context)
 192 {
 193   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (element);
 194
 195   gst_vulkan_handle_set_context (element, context, NULL, &render->instance);
 196
 197   GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
 198 }
 199
 200 struct choose_data
 201 {
 202   GstVulkanFullScreenRender *upload;
 203   GstVulkanQueue *queue;
 204 };
 205
 206 static gboolean
 207 _choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue,
 208     struct choose_data *data)
 209 {
 210   guint flags =
 211       device->physical_device->queue_family_props[queue->family].queueFlags;
 212
 213   GST_ERROR ("flags 0x%x", flags);
 214
 215   if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) {
 216     if (data->queue)
 217       gst_object_unref (data->queue);
 218     data->queue = gst_object_ref (queue);
 219     return FALSE;
 220   }
 221
 222   return TRUE;
 223 }
 224
 225 static GstVulkanQueue *
 226 _find_graphics_queue (GstVulkanFullScreenRender * upload)
 227 {
 228   struct choose_data data;
 229
 230   data.upload = upload;
 231   data.queue = NULL;
 232
 233   gst_vulkan_device_foreach_queue (upload->device,
 234       (GstVulkanDeviceForEachQueueFunc) _choose_queue, &data);
 235
 236   return data.queue;
 237 }
 238
 239 static GstCaps *
 240 gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * bt,
 241     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 242 {
 243   GstCaps *result, *tmp;
 244
 245   tmp = gst_caps_copy (caps);
 246
 247   if (filter) {
 248     result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
 249     gst_caps_unref (tmp);
 250   } else {
 251     result = tmp;
 252   }
 253
 254   return result;
 255 }
 256
 257 static void
 258 clear_shader_create_info (GstVulkanFullScreenRender * render)
 259 {
 260   if (render->shader_create_info) {
 261     if (render->destroy_shader_create_info)
 262       render->destroy_shader_create_info (render, render->shader_create_info);
 263   }
 264   render->n_shader_stages = 0;
 265   render->shader_create_info = NULL;
 266   render->destroy_shader_create_info = NULL;
 267 }
 268
 269 static VkPipeline
 270 _create_pipeline (GstVulkanFullScreenRender * render)
 271 {
 272   GstVulkanFullScreenRenderClass *render_class =
 273       GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
 274   VkVertexInputBindingDescription vertex_binding_description;
 275   VkVertexInputAttributeDescription vertex_attribute_description[2];
 276   VkPipelineVertexInputStateCreateInfo vertex_input_info;
 277   VkPipelineInputAssemblyStateCreateInfo input_assembly;
 278   VkPipelineViewportStateCreateInfo viewport_state;
 279   VkPipelineRasterizationStateCreateInfo rasterizer;
 280   VkPipelineMultisampleStateCreateInfo multisampling;
 281   VkPipelineColorBlendAttachmentState
 282       color_blend_attachments[GST_VIDEO_MAX_PLANES];
 283   VkPipelineColorBlendStateCreateInfo color_blending;
 284   VkGraphicsPipelineCreateInfo pipeline_info;
 285   VkPipeline pipeline;
 286   GError *error = NULL;
 287   VkResult err;
 288
 289   render_class->shader_create_info (render);
 290
 291   /* *INDENT-OFF* */
 292   vertex_binding_description = (VkVertexInputBindingDescription) {
 293       .binding = 0,
 294       .stride = sizeof (struct Vertex),
 295       .inputRate = VK_VERTEX_INPUT_RATE_VERTEX
 296   };
 297
 298   vertex_attribute_description[0] = (VkVertexInputAttributeDescription) {
 299       .binding = 0,
 300       .location = 0,
 301       .format = VK_FORMAT_R32G32B32_SFLOAT,
 302       .offset = G_STRUCT_OFFSET (struct Vertex, x)
 303   };
 304   vertex_attribute_description[1] = (VkVertexInputAttributeDescription) {
 305       .binding = 0,
 306       .location = 1,
 307       .format = VK_FORMAT_R32G32_SFLOAT,
 308       .offset = G_STRUCT_OFFSET (struct Vertex, s)
 309   };
 310
 311   vertex_input_info = (VkPipelineVertexInputStateCreateInfo) {
 312       .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
 313       .pNext = NULL,
 314       .vertexBindingDescriptionCount = 1,
 315       .pVertexBindingDescriptions = &vertex_binding_description,
 316       .vertexAttributeDescriptionCount = G_N_ELEMENTS (vertex_attribute_description),
 317       .pVertexAttributeDescriptions = vertex_attribute_description
 318   };
 319
 320   input_assembly = (VkPipelineInputAssemblyStateCreateInfo) {
 321       .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
 322       .pNext = NULL,
 323       .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST,
 324       .primitiveRestartEnable = VK_FALSE
 325   };
 326
 327   viewport_state = (VkPipelineViewportStateCreateInfo) {
 328       .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
 329       .pNext = NULL,
 330       .viewportCount = 1,
 331       .pViewports = &(VkViewport) {
 332           .x = 0.0f,
 333           .y = 0.0f,
 334           .width = (float) GST_VIDEO_INFO_WIDTH (&render->out_info),
 335           .height = (float) GST_VIDEO_INFO_HEIGHT (&render->out_info),
 336           .minDepth = 0.0f,
 337           .maxDepth = 1.0f
 338       },
 339       .scissorCount = 1,
 340       .pScissors = &(VkRect2D) {
 341           .offset = { 0, 0 },
 342           .extent = {
 343               GST_VIDEO_INFO_WIDTH (&render->out_info),
 344               GST_VIDEO_INFO_HEIGHT (&render->out_info)
 345           }
 346       }
 347   };
 348
 349   rasterizer = (VkPipelineRasterizationStateCreateInfo) {
 350       .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
 351       .pNext = NULL,
 352       .depthClampEnable = VK_FALSE,
 353       .rasterizerDiscardEnable = VK_FALSE,
 354       .polygonMode = VK_POLYGON_MODE_FILL,
 355       .lineWidth = 1.0f,
 356       .cullMode = VK_CULL_MODE_BACK_BIT,
 357       .frontFace = VK_FRONT_FACE_CLOCKWISE,
 358       .depthBiasEnable = VK_FALSE
 359   };
 360
 361   multisampling = (VkPipelineMultisampleStateCreateInfo) {
 362       .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
 363       .pNext = NULL,
 364       .sampleShadingEnable = VK_FALSE,
 365       .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT
 366   };
 367
 368   color_blend_attachments[0] = (VkPipelineColorBlendAttachmentState) {
 369       .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
 370       .blendEnable = VK_FALSE
 371   };
 372   color_blend_attachments[1] = (VkPipelineColorBlendAttachmentState) {
 373       .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
 374       .blendEnable = VK_FALSE
 375   };
 376   color_blend_attachments[2] = (VkPipelineColorBlendAttachmentState) {
 377       .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
 378       .blendEnable = VK_FALSE
 379   };
 380   color_blend_attachments[3] = (VkPipelineColorBlendAttachmentState) {
 381       .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
 382       .blendEnable = VK_FALSE
 383   };
 384
 385   color_blending = (VkPipelineColorBlendStateCreateInfo) {
 386       .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 387       .pNext = NULL,
 388       .logicOpEnable = VK_FALSE,
 389       .logicOp = VK_LOGIC_OP_COPY,
 390       .attachmentCount = GST_VIDEO_INFO_N_PLANES (&render->out_info),
 391       .pAttachments = color_blend_attachments,
 392       .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f }
 393   };
 394
 395   pipeline_info = (VkGraphicsPipelineCreateInfo) {
 396       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
 397       .pNext = NULL,
 398       .stageCount = render->n_shader_stages,
 399       .pStages = render->shader_create_info,
 400       .pVertexInputState = &vertex_input_info,
 401       .pInputAssemblyState = &input_assembly,
 402       .pViewportState = &viewport_state,
 403       .pRasterizationState = &rasterizer,
 404       .pMultisampleState = &multisampling,
 405       .pColorBlendState = &color_blending,
 406       .layout = render->pipeline_layout,
 407       .renderPass = render->render_pass,
 408       .subpass = 0,
 409       .basePipelineHandle = VK_NULL_HANDLE
 410   };
 411   /* *INDENT-ON* */
 412
 413   err =
 414       vkCreateGraphicsPipelines (render->device->device, VK_NULL_HANDLE, 1,
 415       &pipeline_info, NULL, &pipeline);
 416   clear_shader_create_info (render);
 417   if (gst_vulkan_error_to_g_error (err, &error,
 418           "vkCreateGraphicsPipelines") < 0) {
 419     GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s",
 420         error->message);
 421     g_clear_error (&error);
 422     return NULL;
 423   }
 424
 425   return pipeline;
 426 }
 427
 428 static VkPipelineLayout
 429 _create_pipeline_layout (GstVulkanFullScreenRender * render)
 430 {
 431   GstVulkanFullScreenRenderClass *render_class =
 432       GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
 433   VkPipelineLayoutCreateInfo pipeline_layout_info;
 434   VkPipelineLayout pipeline_layout;
 435   VkPushConstantRange *constants = NULL;
 436   guint n_constants = 0;
 437   GError *error = NULL;
 438   VkResult err;
 439
 440   if (render_class->push_constant_ranges)
 441     constants = render_class->push_constant_ranges (render, &n_constants);
 442
 443   /* *INDENT-OFF* */
 444   pipeline_layout_info = (VkPipelineLayoutCreateInfo) {
 445       .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
 446       .pNext = NULL,
 447       .setLayoutCount = 1,
 448       .pSetLayouts = &render->descriptor_set_layout,
 449       .pushConstantRangeCount = n_constants,
 450       .pPushConstantRanges = constants,
 451   };
 452   /* *INDENT-ON* */
 453
 454   err =
 455       vkCreatePipelineLayout (render->device->device,
 456       &pipeline_layout_info, NULL, &pipeline_layout);
 457   g_free (constants);
 458   if (gst_vulkan_error_to_g_error (err, &error, "vkCreatePipelineLayout") < 0) {
 459     GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s",
 460         error->message);
 461     g_clear_error (&error);
 462     return NULL;
 463   }
 464
 465   return pipeline_layout;
 466 }
 467
 468 static VkRenderPass
 469 _create_render_pass (GstVulkanFullScreenRender * render)
 470 {
 471   GstVulkanFullScreenRenderClass *render_class =
 472       GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
 473
 474   guint n_descriptions;
 475   VkAttachmentDescription *descriptions =
 476       render_class->render_pass_attachment_descriptions (render,
 477       &n_descriptions);
 478
 479   guint n_refs;
 480   VkAttachmentReference *color_attachment_refs =
 481       render_class->render_pass_attachment_references (render, &n_refs);
 482
 483   /* *INDENT-OFF* */
 484   VkSubpassDescription subpass = {
 485       .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
 486       .colorAttachmentCount = n_refs,
 487       .pColorAttachments = color_attachment_refs
 488   };
 489
 490   VkRenderPassCreateInfo render_pass_info = {
 491       .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
 492       .pNext = NULL,
 493       .attachmentCount = n_descriptions,
 494       .pAttachments = descriptions,
 495       .subpassCount = 1,
 496       .pSubpasses = &subpass
 497   };
 498   /* *INDENT-ON* */
 499   VkRenderPass render_pass = NULL;
 500   VkResult err;
 501   GError *error = NULL;
 502
 503   err =
 504       vkCreateRenderPass (render->device->device, &render_pass_info, NULL,
 505       &render_pass);
 506   g_free (color_attachment_refs);
 507   g_free (descriptions);
 508   if (gst_vulkan_error_to_g_error (err, &error, "vkCreateRenderPass") < 0) {
 509     GST_ERROR_OBJECT (render, "Failed to create renderpass: %s",
 510         error->message);
 511     return NULL;
 512   }
 513
 514   return render_pass;
 515 }
 516
 517 static VkDescriptorSetLayout
 518 _create_descriptor_set_layout (GstVulkanFullScreenRender * render)
 519 {
 520   GstVulkanFullScreenRenderClass *render_class =
 521       GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
 522   guint n_bindings;
 523   VkDescriptorSetLayoutBinding *bindings =
 524       render_class->descriptor_set_layout_bindings (render, &n_bindings);
 525
 526   /* *INDENT-OFF* */
 527   VkDescriptorSetLayoutCreateInfo layout_info = {
 528       .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
 529       .pNext = NULL,
 530       .bindingCount = n_bindings,
 531       .pBindings = bindings
 532   };
 533   /* *INDENT-ON* */
 534   VkDescriptorSetLayout descriptor_set_layout;
 535   VkResult err;
 536   GError *error = NULL;
 537
 538   err =
 539       vkCreateDescriptorSetLayout (render->device->device, &layout_info,
 540       NULL, &descriptor_set_layout);
 541   g_free (bindings);
 542   if (gst_vulkan_error_to_g_error (err, &error,
 543           "vkCreateDescriptorSetLayout") < 0) {
 544     GST_ERROR_OBJECT (render, "Failed to create renderpass: %s",
 545         error->message);
 546     return NULL;
 547   }
 548
 549   return descriptor_set_layout;
 550 }
 551
 552 static gboolean
 553 gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt,
 554     GstCaps * in_caps, GstCaps * out_caps)
 555 {
 556   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
 557
 558   if (!gst_video_info_from_caps (&render->in_info, in_caps))
 559     return FALSE;
 560   if (!gst_video_info_from_caps (&render->out_info, out_caps))
 561     return FALSE;
 562
 563   gst_caps_replace (&render->in_caps, in_caps);
 564   gst_caps_replace (&render->out_caps, out_caps);
 565
 566   if (render->last_fence) {
 567     if (render->descriptor_set_layout) {
 568       gst_vulkan_trash_list_add (render->trash_list,
 569           gst_vulkan_trash_new_free_descriptor_set_layout (gst_vulkan_fence_ref
 570               (render->last_fence), render->descriptor_set_layout));
 571       render->descriptor_set_layout = NULL;
 572     }
 573     if (render->pipeline_layout) {
 574       gst_vulkan_trash_list_add (render->trash_list,
 575           gst_vulkan_trash_new_free_pipeline_layout (gst_vulkan_fence_ref
 576               (render->last_fence), render->pipeline_layout));
 577       render->pipeline_layout = NULL;
 578     }
 579     if (render->render_pass) {
 580       gst_vulkan_trash_list_add (render->trash_list,
 581           gst_vulkan_trash_new_free_render_pass (gst_vulkan_fence_ref
 582               (render->last_fence), render->render_pass));
 583       render->render_pass = NULL;
 584     }
 585     if (render->graphics_pipeline) {
 586       gst_vulkan_trash_list_add (render->trash_list,
 587           gst_vulkan_trash_new_free_pipeline (gst_vulkan_fence_ref
 588               (render->last_fence), render->graphics_pipeline));
 589       render->graphics_pipeline = NULL;
 590     }
 591   } else {
 592     if (render->graphics_pipeline)
 593       vkDestroyPipeline (render->device->device,
 594           render->graphics_pipeline, NULL);
 595     render->graphics_pipeline = NULL;
 596
 597     if (render->pipeline_layout)
 598       vkDestroyPipelineLayout (render->device->device,
 599           render->pipeline_layout, NULL);
 600     render->pipeline_layout = NULL;
 601
 602     if (render->render_pass)
 603       vkDestroyRenderPass (render->device->device, render->render_pass, NULL);
 604     render->render_pass = NULL;
 605
 606     if (render->descriptor_set_layout)
 607       vkDestroyDescriptorSetLayout (render->device->device,
 608           render->descriptor_set_layout, NULL);
 609     render->descriptor_set_layout = NULL;
 610   }
 611
 612   if (!(render->descriptor_set_layout = _create_descriptor_set_layout (render)))
 613     return FALSE;
 614   if (!(render->pipeline_layout = _create_pipeline_layout (render)))
 615     return FALSE;
 616   if (!(render->render_pass = _create_render_pass (render)))
 617     return FALSE;
 618   if (!(render->graphics_pipeline = _create_pipeline (render)))
 619     return FALSE;
 620
 621   GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps);
 622
 623   return TRUE;
 624 }
 625
 626 static gboolean
 627 gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt,
 628     GstQuery * decide_query, GstQuery * query)
 629 {
 630   /* FIXME: */
 631   return FALSE;
 632 }
 633
 634 static gboolean
 635 gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt,
 636     GstQuery * query)
 637 {
 638   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
 639   GstBufferPool *pool = NULL;
 640   GstStructure *config;
 641   GstCaps *caps;
 642   guint min, max, size;
 643   gboolean update_pool;
 644
 645   gst_query_parse_allocation (query, &caps, NULL);
 646   if (!caps)
 647     return FALSE;
 648
 649   if (gst_query_get_n_allocation_pools (query) > 0) {
 650     gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
 651
 652     update_pool = TRUE;
 653   } else {
 654     GstVideoInfo vinfo;
 655
 656     gst_video_info_init (&vinfo);
 657     gst_video_info_from_caps (&vinfo, caps);
 658     size = vinfo.size;
 659     min = max = 0;
 660     update_pool = FALSE;
 661   }
 662
 663   if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) {
 664     if (pool)
 665       gst_object_unref (pool);
 666     pool = gst_vulkan_image_buffer_pool_new (render->device);
 667   }
 668
 669   config = gst_buffer_pool_get_config (pool);
 670
 671   gst_buffer_pool_config_set_params (config, caps, size, min, max);
 672   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
 673
 674   gst_buffer_pool_set_config (pool, config);
 675
 676   if (update_pool)
 677     gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
 678   else
 679     gst_query_add_allocation_pool (query, pool, size, min, max);
 680
 681   gst_object_unref (pool);
 682
 683   return TRUE;
 684 }
 685
 686 static gboolean
 687 _create_vertex_buffers (GstVulkanFullScreenRender * render)
 688 {
 689   GstMapInfo map_info;
 690
 691   render->vertices =
 692       gst_vulkan_buffer_memory_alloc (render->device, sizeof (vertices),
 693       VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
 694       VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 695       VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
 696
 697   if (!gst_memory_map (render->vertices, &map_info, GST_MAP_WRITE)) {
 698     gst_memory_unref (render->vertices);
 699     render->vertices = NULL;
 700     return FALSE;
 701   }
 702   memcpy (map_info.data, vertices, sizeof (vertices));
 703   gst_memory_unmap (render->vertices, &map_info);
 704
 705   render->indices =
 706       gst_vulkan_buffer_memory_alloc (render->device, sizeof (indices),
 707       VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT,
 708       VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 709       VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
 710
 711   if (!gst_memory_map (render->indices, &map_info, GST_MAP_WRITE)) {
 712     gst_memory_unref (render->vertices);
 713     render->vertices = NULL;
 714     gst_memory_unref (render->indices);
 715     render->indices = NULL;
 716     return FALSE;
 717   }
 718   memcpy (map_info.data, indices, sizeof (indices));
 719   gst_memory_unmap (render->indices, &map_info);
 720
 721   return TRUE;
 722 }
 723
 724 static gboolean
 725 gst_vulkan_full_screen_render_start (GstBaseTransform * bt)
 726 {
 727   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
 728
 729   if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL,
 730           &render->instance)) {
 731     GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
 732         ("Failed to retrieve vulkan instance"), (NULL));
 733     return FALSE;
 734   }
 735   if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render),
 736           &render->device)) {
 737     GError *error = NULL;
 738     GST_DEBUG_OBJECT (render, "No device retrieved from peer elements");
 739     if (!(render->device =
 740             gst_vulkan_instance_create_device (render->instance, &error))) {
 741       GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
 742           ("Failed to create vulkan device"), ("%s", error->message));
 743       g_clear_error (&error);
 744       return FALSE;
 745     }
 746   }
 747
 748   if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render),
 749           &render->queue)) {
 750     GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements");
 751     render->queue = _find_graphics_queue (render);
 752   }
 753   if (!render->queue)
 754     return FALSE;
 755
 756   if (!_create_vertex_buffers (render))
 757     return FALSE;
 758
 759   render->trash_list = gst_vulkan_trash_fence_list_new ();
 760
 761   return TRUE;
 762 }
 763
 764 static gboolean
 765 gst_vulkan_full_screen_render_stop (GstBaseTransform * bt)
 766 {
 767   GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
 768
 769   if (render->device) {
 770     if (render->last_fence) {
 771       gst_vulkan_trash_list_add (render->trash_list,
 772           gst_vulkan_trash_new_free_pipeline (gst_vulkan_fence_ref
 773               (render->last_fence), render->graphics_pipeline));
 774       render->graphics_pipeline = NULL;
 775       gst_vulkan_trash_list_add (render->trash_list,
 776           gst_vulkan_trash_new_free_pipeline_layout (gst_vulkan_fence_ref
 777               (render->last_fence), render->pipeline_layout));
 778       render->pipeline_layout = NULL;
 779       gst_vulkan_trash_list_add (render->trash_list,
 780           gst_vulkan_trash_new_free_render_pass (gst_vulkan_fence_ref
 781               (render->last_fence), render->render_pass));
 782       render->render_pass = NULL;
 783       gst_vulkan_trash_list_add (render->trash_list,
 784           gst_vulkan_trash_new_free_descriptor_set_layout (gst_vulkan_fence_ref
 785               (render->last_fence), render->descriptor_set_layout));
 786       render->descriptor_set_layout = NULL;
 787
 788       gst_vulkan_fence_unref (render->last_fence);
 789       render->last_fence = NULL;
 790     } else {
 791       vkDestroyPipeline (render->device->device,
 792           render->graphics_pipeline, NULL);
 793       render->graphics_pipeline = NULL;
 794
 795       vkDestroyPipelineLayout (render->device->device,
 796           render->pipeline_layout, NULL);
 797       render->pipeline_layout = NULL;
 798
 799       vkDestroyRenderPass (render->device->device, render->render_pass, NULL);
 800       render->render_pass = NULL;
 801
 802       vkDestroyDescriptorSetLayout (render->device->device,
 803           render->descriptor_set_layout, NULL);
 804       render->descriptor_set_layout = NULL;
 805     }
 806
 807     if (!gst_vulkan_trash_list_wait (render->trash_list, -1))
 808       GST_WARNING_OBJECT (render,
 809           "Failed to wait for all resources to be freed");
 810     gst_object_unref (render->trash_list);
 811     render->trash_list = NULL;
 812
 813     if (render->vertices)
 814       gst_memory_unref (render->vertices);
 815     render->vertices = NULL;
 816
 817     if (render->indices)
 818       gst_memory_unref (render->indices);
 819     render->indices = NULL;
 820
 821     gst_object_unref (render->device);
 822   }
 823   render->device = NULL;
 824
 825   if (render->queue)
 826     gst_object_unref (render->queue);
 827   render->queue = NULL;
 828
 829   if (render->instance)
 830     gst_object_unref (render->instance);
 831   render->instance = NULL;
 832
 833   return TRUE;
 834 }
 835
 836 /**
 837  * gst_vulkan_full_screen_render_fill_command_buffer:
 838  * @render: a #GstVulkanFullScreenRender
 839  * @cmd: a `VkCommandBuffer`
 840  * @framebuffer: a `VkFramebuffer`
 841  *
 842  * Returns: whether @cmd could be filled with the commands necessary to render
 843  */
 844 gboolean
 845 gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender *
 846     render, VkCommandBuffer cmd, VkFramebuffer framebuffer)
 847 {
 848   /* *INDENT-OFF* */
 849   VkClearValue clearColor = {{{ 0.0f, 0.0f, 0.0f, 1.0f }}};
 850   VkClearValue clearColors[GST_VIDEO_MAX_PLANES] = {
 851     clearColor, clearColor, clearColor, clearColor,
 852   };
 853   VkRenderPassBeginInfo render_pass_info = {
 854       .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 855       .renderPass = render->render_pass,
 856       .framebuffer = framebuffer,
 857       .renderArea.offset = { 0, 0 },
 858       .renderArea.extent = {
 859           GST_VIDEO_INFO_WIDTH (&render->out_info),
 860           GST_VIDEO_INFO_HEIGHT (&render->out_info)
 861       },
 862       .clearValueCount = GST_VIDEO_INFO_N_PLANES (&render->out_info),
 863       .pClearValues = clearColors,
 864   };
 865   /* *INDENI-ON* */
 866   VkDeviceSize offsets[] = { 0 };
 867
 868   vkCmdBeginRenderPass (cmd, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE);
 869   vkCmdBindPipeline (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
 870       render->graphics_pipeline);
 871   vkCmdBindVertexBuffers (cmd, 0, 1,
 872       &((GstVulkanBufferMemory *) render->vertices)->buffer, offsets);
 873   vkCmdBindIndexBuffer (cmd,
 874       ((GstVulkanBufferMemory *) render->indices)->buffer, 0,
 875       VK_INDEX_TYPE_UINT16);
 876   vkCmdDrawIndexed (cmd, G_N_ELEMENTS (indices), 1, 0, 0, 0);
 877   vkCmdEndRenderPass (cmd);
 878
 879   return TRUE;
 880 }
 881
 882 gboolean
 883 gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render,
 884     VkCommandBuffer cmd, GstVulkanFence * fence)
 885 {
 886   VkSubmitInfo submit_info;
 887   GError *error = NULL;
 888   VkResult err;
 889
 890   if (!fence)
 891     fence = gst_vulkan_fence_new (render->device, 0, &error);
 892   if (!fence)
 893     goto error;
 894
 895   /* *INDENT-OFF* */
 896   submit_info = (VkSubmitInfo) {
 897       .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
 898       .pNext = NULL,
 899       .waitSemaphoreCount = 0,
 900       .pWaitSemaphores = NULL,
 901       .pWaitDstStageMask = NULL,
 902       .commandBufferCount = 1,
 903       .pCommandBuffers = &cmd,
 904       .signalSemaphoreCount = 0,
 905       .pSignalSemaphores = NULL,
 906   };
 907   /* *INDENT-ON* */
 908
 909   if (render->last_fence)
 910     gst_vulkan_fence_unref (render->last_fence);
 911   render->last_fence = gst_vulkan_fence_ref (fence);
 912
 913   err =
 914       vkQueueSubmit (render->queue->queue, 1, &submit_info,
 915       GST_VULKAN_FENCE_FENCE (fence));
 916   if (gst_vulkan_error_to_g_error (err, &error, "vkQueueSubmit") < 0)
 917     goto error;
 918
 919   gst_vulkan_trash_list_gc (render->trash_list);
 920
 921   gst_vulkan_fence_unref (fence);
 922
 923   return TRUE;
 924
 925 error:
 926   GST_ELEMENT_ERROR (render, LIBRARY, FAILED, ("%s", error->message), (NULL));
 927   g_clear_error (&error);
 928   return FALSE;
 929 }