3 * Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-vulkanoverlaycompositor
23 * @title: vulkanoverlaycompositor
25 * `vulkanoverlaycompositor` overlays upstream `GstVideoOverlayCompositonMeta`
26 * onto the video stream.
37 #include "gstvulkanelements.h"
38 #include "vkoverlaycompositor.h"
40 #include "shaders/identity.vert.h"
41 #include "shaders/swizzle.frag.h"
43 GST_DEBUG_CATEGORY (gst_debug_vulkan_overlay_compositor);
44 #define GST_CAT_DEFAULT gst_debug_vulkan_overlay_compositor
49 GstVideoOverlayComposition *composition;
50 GstVideoOverlayRectangle *rectangle;
51 GstVulkanFullScreenQuad *quad;
55 vk_overlay_clear (struct vk_overlay *overlay)
57 gst_clear_buffer (&overlay->buffer);
58 overlay->rectangle = NULL;
59 if (overlay->composition)
60 gst_video_overlay_composition_unref (overlay->composition);
61 overlay->composition = NULL;
63 gst_clear_object (&overlay->quad);
67 vk_overlay_init (struct vk_overlay *overlay, GstVulkanQueue * queue,
68 GstBuffer * buffer, GstVideoOverlayComposition * comp,
69 GstVideoOverlayRectangle * rectangle, GstVulkanHandle * vert,
70 GstVulkanHandle * frag)
72 GstVideoOverlayFormatFlags flags;
74 memset (overlay, 0, sizeof (*overlay));
76 flags = gst_video_overlay_rectangle_get_flags (rectangle);
78 overlay->buffer = gst_buffer_ref (buffer);
79 overlay->composition = gst_video_overlay_composition_ref (comp);
80 overlay->rectangle = rectangle;
81 overlay->quad = gst_vulkan_full_screen_quad_new (queue);
82 gst_vulkan_full_screen_quad_enable_clear (overlay->quad, FALSE);
83 gst_vulkan_full_screen_quad_set_shaders (overlay->quad, vert, frag);
84 gst_vulkan_full_screen_quad_enable_blend (overlay->quad, TRUE);
85 gst_vulkan_full_screen_quad_set_blend_operation (overlay->quad,
86 VK_BLEND_OP_ADD, VK_BLEND_OP_ADD);
87 if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA) {
88 gst_vulkan_full_screen_quad_set_blend_factors (overlay->quad,
89 VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
90 VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA);
92 gst_vulkan_full_screen_quad_set_blend_factors (overlay->quad,
93 VK_BLEND_FACTOR_SRC_ALPHA, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
94 VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA);
104 struct swizzle_uniforms
106 int in_reorder_index[4];
107 int out_reorder_index[4];
111 vk_overlay_upload (struct vk_overlay *overlay, GstVideoInfo * out_info,
114 GstBuffer *overlay_buffer, *vk_gst_buffer = NULL;
117 GstVideoFrame vframe;
118 GstVulkanBufferMemory *buf_mem;
119 GstVulkanImageMemory *img_mem;
120 GstMemory *vkbuffer = NULL, *vkimage = NULL, *vkvertices = NULL;
121 GstMemory *vkuniforms = NULL;
124 GstVulkanFence *fence = NULL;
125 GstVulkanCommandBuffer *cmd_buf = NULL;
126 VkBufferMemoryBarrier buffer_memory_barrier;
127 VkImageMemoryBarrier image_memory_barrier;
128 VkBufferImageCopy region;
130 struct Vertex vertices[4];
131 struct swizzle_uniforms uniforms;
134 gst_video_overlay_rectangle_get_pixels_unscaled_argb
135 (overlay->rectangle, GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE);
137 vmeta = gst_buffer_get_video_meta (overlay_buffer);
138 gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width,
140 vinfo.stride[0] = vmeta->stride[0];
142 if (!gst_vulkan_full_screen_quad_set_info (overlay->quad, out_info, out_info))
145 if (!gst_video_frame_map (&vframe, &vinfo, overlay_buffer, GST_MAP_READ)) {
146 g_set_error_literal (error, GST_TYPE_RESOURCE_ERROR,
147 GST_RESOURCE_ERROR_READ, "Cannot map overlay buffer for reading");
152 gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
153 GST_VIDEO_INFO_COMP_STRIDE (&vinfo, 0) *
154 GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
155 VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
156 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
157 buf_mem = (GstVulkanBufferMemory *) vkbuffer;
159 if (!gst_memory_map (vkbuffer, &map_info, GST_MAP_WRITE)) {
160 g_set_error_literal (error, GST_TYPE_RESOURCE_ERROR,
161 GST_RESOURCE_ERROR_WRITE,
162 "Cannot map staging vulkan buffer for writing");
163 gst_video_frame_unmap (&vframe);
167 memcpy (map_info.data, vframe.data[0], vframe.info.size);
169 gst_memory_unmap (vkbuffer, &map_info);
170 gst_video_frame_unmap (&vframe);
172 vk_format = gst_vulkan_format_from_video_info (&vinfo, 0);
174 gst_vulkan_image_memory_alloc (overlay->quad->queue->device, vk_format,
175 GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
176 GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
177 VK_IMAGE_TILING_OPTIMAL,
178 VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT |
179 VK_IMAGE_USAGE_SAMPLED_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
180 img_mem = (GstVulkanImageMemory *) vkimage;
183 region = (VkBufferImageCopy) {
185 .bufferRowLength = GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
186 .bufferImageHeight = GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
187 .imageSubresource = {
188 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
193 .imageOffset = { .x = 0, .y = 0, .z = 0, },
195 .width = GST_VIDEO_INFO_COMP_WIDTH (&vinfo, 0),
196 .height = GST_VIDEO_INFO_COMP_HEIGHT (&vinfo, 0),
201 buffer_memory_barrier = (VkBufferMemoryBarrier) {
202 .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
204 .srcAccessMask = buf_mem->barrier.parent.access_flags,
205 .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
206 /* FIXME: implement exclusive transfers */
207 .srcQueueFamilyIndex = 0,
208 .dstQueueFamilyIndex = 0,
209 .buffer = buf_mem->buffer,
210 .offset = region.bufferOffset,
211 .size = region.bufferRowLength * region.bufferImageHeight,
214 image_memory_barrier = (VkImageMemoryBarrier) {
215 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
217 .srcAccessMask = img_mem->barrier.parent.access_flags,
218 .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
219 .oldLayout = img_mem->barrier.image_layout,
220 .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
221 /* FIXME: implement exclusive transfers */
222 .srcQueueFamilyIndex = 0,
223 .dstQueueFamilyIndex = 0,
224 .image = img_mem->image,
225 .subresourceRange = img_mem->barrier.subresource_range,
230 gst_vulkan_command_pool_create (overlay->quad->cmd_pool, error)))
235 VkCommandBufferBeginInfo cmd_buf_info = {
236 .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
238 .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
239 .pInheritanceInfo = NULL
243 gst_vulkan_command_buffer_lock (cmd_buf);
244 err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
245 if (gst_vulkan_error_to_g_error (err, error, "vkBeginCommandBuffer") < 0) {
246 gst_vulkan_command_buffer_unlock (cmd_buf);
251 vkCmdPipelineBarrier (cmd_buf->cmd,
252 buf_mem->barrier.parent.pipeline_stages | img_mem->barrier.
253 parent.pipeline_stages, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 1,
254 &buffer_memory_barrier, 1, &image_memory_barrier);
256 buf_mem->barrier.parent.pipeline_stages = VK_PIPELINE_STAGE_TRANSFER_BIT;
257 buf_mem->barrier.parent.access_flags = buffer_memory_barrier.dstAccessMask;
259 img_mem->barrier.parent.pipeline_stages = VK_PIPELINE_STAGE_TRANSFER_BIT;
260 img_mem->barrier.parent.access_flags = image_memory_barrier.dstAccessMask;
261 img_mem->barrier.image_layout = image_memory_barrier.newLayout;
263 vkCmdCopyBufferToImage (cmd_buf->cmd, buf_mem->buffer, img_mem->image,
264 img_mem->barrier.image_layout, 1, ®ion);
266 err = vkEndCommandBuffer (cmd_buf->cmd);
267 gst_vulkan_command_buffer_unlock (cmd_buf);
268 if (gst_vulkan_error_to_g_error (err, error, "vkEndCommandBuffer") < 0) {
273 VkSubmitInfo submit_info = { 0, };
276 submit_info = (VkSubmitInfo) {
277 .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
279 .waitSemaphoreCount = 0,
280 .pWaitSemaphores = NULL,
281 .pWaitDstStageMask = NULL,
282 .commandBufferCount = 1,
283 .pCommandBuffers = &cmd_buf->cmd,
284 .signalSemaphoreCount = 0,
285 .pSignalSemaphores = NULL,
290 gst_vulkan_device_create_fence (overlay->quad->queue->device, error);
294 gst_vulkan_queue_submit_lock (overlay->quad->queue);
296 vkQueueSubmit (overlay->quad->queue->queue, 1, &submit_info,
297 GST_VULKAN_FENCE_FENCE (fence));
298 gst_vulkan_queue_submit_unlock (overlay->quad->queue);
299 if (gst_vulkan_error_to_g_error (err, error, "vkQueueSubmit") < 0)
302 gst_vulkan_trash_list_add (overlay->quad->trash_list,
303 gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
304 gst_vulkan_trash_mini_object_unref,
305 GST_MINI_OBJECT_CAST (cmd_buf)));
307 gst_vulkan_trash_list_add (overlay->quad->trash_list,
308 gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
309 gst_vulkan_trash_mini_object_unref,
310 GST_MINI_OBJECT_CAST (vkbuffer)));
312 gst_vulkan_trash_list_add (overlay->quad->trash_list,
313 gst_vulkan_trash_list_acquire (overlay->quad->trash_list, fence,
314 gst_vulkan_trash_mini_object_unref,
315 GST_MINI_OBJECT_CAST (gst_memory_ref (vkimage))));
316 gst_vulkan_trash_list_gc (overlay->quad->trash_list);
317 gst_vulkan_fence_unref (fence);
321 vk_gst_buffer = gst_buffer_new ();
322 gst_buffer_append_memory (vk_gst_buffer, vkimage);
325 if (!gst_vulkan_full_screen_quad_set_input_buffer (overlay->quad,
326 vk_gst_buffer, error))
329 gst_clear_buffer (&vk_gst_buffer);
333 guint width, height, out_width, out_height;
334 float xl, xr, yt, yb;
336 if (!gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
337 &xpos, &ypos, &width, &height))
340 out_width = GST_VIDEO_INFO_WIDTH (out_info);
341 out_height = GST_VIDEO_INFO_HEIGHT (out_info);
343 xl = 2.0 * (float) xpos / (float) out_width - 1.0;
344 yt = 2.0 * (float) ypos / (float) out_height - 1.0;
345 xr = xl + 2.0 * (float) width / (float) out_width;
346 yb = yt + 2.0 * (float) height / (float) out_height;
348 GST_LOG_OBJECT (overlay->quad, "rectangle %ux%u+%d,%d placed in %ux%u at "
349 "%fx%f+%f,%f", width, height, xpos, ypos, out_width, out_height,
350 xr - xl, yb - yt, xl, yt);
379 gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
381 VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
382 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
383 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
385 if (!gst_memory_map (vkvertices, &map_info, GST_MAP_WRITE))
387 memcpy (map_info.data, vertices, sizeof (vertices));
388 gst_memory_unmap (vkvertices, &map_info);
390 if (!gst_vulkan_full_screen_quad_set_vertex_buffer (overlay->quad,
394 gst_clear_mini_object ((GstMiniObject **) & vkvertices);
396 uniforms.in_reorder_index[0] = 0;
397 uniforms.in_reorder_index[1] = 1;
398 uniforms.in_reorder_index[2] = 2;
399 uniforms.in_reorder_index[3] = 3;
400 uniforms.out_reorder_index[0] = 0;
401 uniforms.out_reorder_index[1] = 1;
402 uniforms.out_reorder_index[2] = 2;
403 uniforms.out_reorder_index[3] = 3;
406 gst_vulkan_buffer_memory_alloc (overlay->quad->queue->device,
408 VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
409 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
410 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
412 if (!gst_memory_map (vkuniforms, &map_info, GST_MAP_WRITE))
414 memcpy (map_info.data, &uniforms, sizeof (uniforms));
415 gst_memory_unmap (vkuniforms, &map_info);
417 if (!gst_vulkan_full_screen_quad_set_uniform_buffer (overlay->quad,
420 gst_clear_mini_object ((GstMiniObject **) & vkuniforms);
425 gst_clear_mini_object ((GstMiniObject **) & vkimage);
426 gst_clear_mini_object ((GstMiniObject **) & vkbuffer);
427 gst_clear_mini_object ((GstMiniObject **) & vkvertices);
428 gst_clear_mini_object ((GstMiniObject **) & vkuniforms);
430 gst_vulkan_command_buffer_unref (cmd_buf);
432 gst_vulkan_fence_unref (fence);
433 gst_clear_buffer (&vk_gst_buffer);
434 gst_clear_buffer (&overlay_buffer);
439 static gboolean gst_vulkan_overlay_compositor_start (GstBaseTransform * bt);
440 static gboolean gst_vulkan_overlay_compositor_stop (GstBaseTransform * bt);
441 static GstCaps *gst_vulkan_overlay_compositor_transform_caps (GstBaseTransform *
442 bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
443 static gboolean gst_vulkan_overlay_compositor_set_caps (GstBaseTransform *
444 bt, GstCaps * incaps, GstCaps * outcaps);
446 gst_vulkan_overlay_compositor_transform_ip (GstBaseTransform * bt,
449 #define IMAGE_FORMATS " { BGRA }"
451 static GstStaticPadTemplate gst_vulkan_sink_template =
452 GST_STATIC_PAD_TEMPLATE ("sink",
455 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
456 (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE ","
457 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
459 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
460 IMAGE_FORMATS) "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
463 static GstStaticPadTemplate gst_vulkan_src_template =
464 GST_STATIC_PAD_TEMPLATE ("src",
467 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
468 (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE ","
469 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
471 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
472 IMAGE_FORMATS) "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY",
480 typedef struct _GstVulkanOverlayCompositor GstVulkanOverlayCompositor;
482 struct _GstVulkanOverlayCompositor
484 GstVulkanVideoFilter parent;
486 GstVulkanHandle *vert;
487 GstVulkanHandle *frag;
490 gboolean render_overlays;
493 #define gst_vulkan_overlay_compositor_parent_class parent_class
494 G_DEFINE_TYPE_WITH_CODE (GstVulkanOverlayCompositor,
495 gst_vulkan_overlay_compositor, GST_TYPE_VULKAN_VIDEO_FILTER,
496 GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_overlay_compositor,
497 "vulkanoverlaycompositor", 0, "Vulkan Overlay Compositor"));
498 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanoverlaycompositor,
499 "vulkanoverlaycompositor", GST_RANK_NONE,
500 GST_TYPE_VULKAN_OVERLAY_COMPOSITOR, vulkan_element_init (plugin));
503 gst_vulkan_overlay_compositor_class_init (GstVulkanOverlayCompositorClass *
506 GstElementClass *gstelement_class;
507 GstBaseTransformClass *gstbasetransform_class;
509 gstelement_class = (GstElementClass *) klass;
510 gstbasetransform_class = (GstBaseTransformClass *) klass;
512 gst_element_class_set_metadata (gstelement_class, "Vulkan Overlay Compositor",
513 "Filter/Video", "Vulkan Overlay Composition element",
514 "Matthew Waters <matthew@centricular.com>");
516 gst_element_class_add_static_pad_template (gstelement_class,
517 &gst_vulkan_sink_template);
518 gst_element_class_add_static_pad_template (gstelement_class,
519 &gst_vulkan_src_template);
521 gstbasetransform_class->start =
522 GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_start);
523 gstbasetransform_class->stop =
524 GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_stop);
525 gstbasetransform_class->transform_caps =
526 GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_transform_caps);
527 gstbasetransform_class->set_caps =
528 GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_set_caps);
529 gstbasetransform_class->transform_ip =
530 GST_DEBUG_FUNCPTR (gst_vulkan_overlay_compositor_transform_ip);
534 gst_vulkan_overlay_compositor_init (GstVulkanOverlayCompositor * vk_overlay)
539 gst_vulkan_overlay_compositor_start (GstBaseTransform * bt)
541 GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
542 GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (vk_overlay);
543 GError *error = NULL;
545 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
548 if (!(vk_overlay->vert =
549 gst_vulkan_create_shader (vfilter->device, identity_vert,
550 identity_vert_size, &error)))
552 if (!(vk_overlay->frag =
553 gst_vulkan_create_shader (vfilter->device, swizzle_frag,
554 swizzle_frag_size, &error))) {
555 gst_clear_vulkan_handle (&vk_overlay->vert);
559 vk_overlay->overlays = g_array_new (FALSE, TRUE, sizeof (struct vk_overlay));
560 g_array_set_clear_func (vk_overlay->overlays,
561 (GDestroyNotify) vk_overlay_clear);
566 GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL));
571 gst_vulkan_overlay_compositor_stop (GstBaseTransform * bt)
573 GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
575 if (vk_overlay->overlays) {
576 g_array_set_size (vk_overlay->overlays, 0);
577 g_array_unref (vk_overlay->overlays);
579 vk_overlay->overlays = NULL;
581 gst_clear_vulkan_handle (&vk_overlay->vert);
582 gst_clear_vulkan_handle (&vk_overlay->frag);
584 return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
587 static struct vk_overlay *
588 find_by_rectangle (GstVulkanOverlayCompositor * vk_overlay,
589 GstVideoOverlayRectangle * rectangle)
593 for (i = 0; i < vk_overlay->overlays->len; i++) {
594 struct vk_overlay *over =
595 &g_array_index (vk_overlay->overlays, struct vk_overlay, i);
597 if (over->rectangle == rectangle)
605 overlay_in_rectangles (struct vk_overlay *over,
606 GstVideoOverlayComposition * composition)
610 n = gst_video_overlay_composition_n_rectangles (composition);
611 for (i = 0; i < n; i++) {
612 GstVideoOverlayRectangle *rect;
614 rect = gst_video_overlay_composition_get_rectangle (composition, i);
616 if (over->rectangle == rect)
624 gst_vulkan_overlay_compositor_transform_caps (GstBaseTransform * bt,
625 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
629 /* add/remove the composition overlay meta as necessary */
630 if (direction == GST_PAD_SRC) {
631 GstCaps *composition_caps;
634 composition_caps = gst_caps_copy (caps);
636 for (i = 0; i < gst_caps_get_size (composition_caps); i++) {
637 GstCapsFeatures *f = gst_caps_get_features (composition_caps, i);
638 if (!gst_caps_features_is_any (f))
639 gst_caps_features_add (f,
640 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
643 ret = gst_caps_merge (composition_caps, gst_caps_copy (caps));
648 ret = gst_caps_copy (caps);
649 removed = gst_caps_copy (caps);
650 n = gst_caps_get_size (removed);
651 for (i = 0; i < n; i++) {
652 GstCapsFeatures *feat = gst_caps_get_features (removed, i);
654 if (feat && gst_caps_features_contains (feat,
655 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
656 feat = gst_caps_features_copy (feat);
657 /* prefer the passthrough case */
658 gst_caps_features_remove (feat,
659 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
660 gst_caps_set_features (removed, i, feat);
664 ret = gst_caps_merge (ret, removed);
668 GstCaps *tmp = gst_caps_intersect (ret, filter);
669 gst_clear_caps (&ret);
677 gst_vulkan_overlay_compositor_set_caps (GstBaseTransform * bt, GstCaps * incaps,
680 GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
681 GstCapsFeatures *in_features, *out_features;
683 GST_DEBUG_OBJECT (bt, " incaps %" GST_PTR_FORMAT, incaps);
684 GST_DEBUG_OBJECT (bt, "outcaps %" GST_PTR_FORMAT, outcaps);
686 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, incaps, outcaps))
689 in_features = gst_caps_get_features (incaps, 0);
690 out_features = gst_caps_get_features (outcaps, 0);
692 if (gst_caps_features_contains (in_features,
693 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION) &&
694 !gst_caps_features_contains (out_features,
695 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) {
696 GST_INFO_OBJECT (bt, "caps say to render GstVideoOverlayCompositionMeta");
697 vk_overlay->render_overlays = TRUE;
700 "caps say to not render GstVideoOverlayCompositionMeta");
701 vk_overlay->render_overlays = FALSE;
708 gst_vulkan_overlay_compositor_transform_ip (GstBaseTransform * bt,
711 GstVulkanOverlayCompositor *vk_overlay = GST_VULKAN_OVERLAY_COMPOSITOR (bt);
712 GstVideoOverlayCompositionMeta *ometa;
713 GstVideoOverlayComposition *comp = NULL;
714 GError *error = NULL;
717 if (!vk_overlay->render_overlays) {
719 "caps don't say to render GstVideoOverlayCompositionMeta, passthrough");
723 ometa = gst_buffer_get_video_overlay_composition_meta (buffer);
726 "no GstVideoOverlayCompositionMeta on buffer, passthrough");
730 comp = gst_video_overlay_composition_ref (ometa->overlay);
731 gst_buffer_remove_meta (buffer, (GstMeta *) ometa);
734 n = gst_video_overlay_composition_n_rectangles (comp);
737 "GstVideoOverlayCompositionMeta has 0 rectangles, passthrough");
742 "rendering GstVideoOverlayCompositionMeta with %u rectangles", n);
743 for (i = 0; i < n; i++) {
744 GstVideoOverlayRectangle *rectangle;
745 struct vk_overlay *over;
747 rectangle = gst_video_overlay_composition_get_rectangle (comp, i);
749 over = find_by_rectangle (vk_overlay, rectangle);
751 struct vk_overlay new_overlay = { 0, };
753 vk_overlay_init (&new_overlay, vk_overlay->parent.queue, buffer, comp,
754 rectangle, vk_overlay->vert, vk_overlay->frag);
756 if (!vk_overlay_upload (&new_overlay, &vk_overlay->parent.out_info,
760 g_array_append_val (vk_overlay->overlays, new_overlay);
764 n = vk_overlay->overlays->len;
765 for (i = 0; i < n;) {
766 struct vk_overlay *over =
767 &g_array_index (vk_overlay->overlays, struct vk_overlay, i);
769 if (!overlay_in_rectangles (over, comp)) {
770 g_array_remove_index (vk_overlay->overlays, i);
774 if (!gst_vulkan_full_screen_quad_set_output_buffer (over->quad, buffer,
778 if (!gst_vulkan_full_screen_quad_draw (over->quad, &error))
785 gst_video_overlay_composition_unref (comp);
790 GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
791 g_clear_error (&error);
793 gst_video_overlay_composition_unref (comp);
794 return GST_FLOW_ERROR;