plugins: implement uploads from raw YUV buffers for GStreamer 1.0.

Implement GstVideoMeta::{,un}map() to support raw YUV buffer upload when
the last component is unmapped. Downloads are not supported yet. The aim
was to first support SW decoding + HW accelerated rendering (vaapisink).
e.g. for Wayland.

diff --git a/gst/vaapi/gstvaapivideobufferpool.c b/gst/vaapi/gstvaapivideobufferpool.c
index 06e8d85..b6002fb 100644 (file)
--- a/gst/vaapi/gstvaapivideobufferpool.c
+++ b/gst/vaapi/gstvaapivideobufferpool.c
@@ -185,10 +185,16 @@ gst_vaapi_video_buffer_pool_alloc_buffer(GstBufferPool *pool,
 
     if (priv->has_video_meta) {
         GstVideoInfo * const vip =
-            &GST_VAAPI_VIDEO_ALLOCATOR_CAST(priv->allocator)->video_info;
-
-        gst_buffer_add_video_meta(buffer, 0, GST_VIDEO_INFO_FORMAT(vip),
-            GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+            &GST_VAAPI_VIDEO_ALLOCATOR_CAST(priv->allocator)->image_info;
+        GstVideoMeta *vmeta;
+
+        vmeta = gst_buffer_add_video_meta_full(buffer, 0,
+            GST_VIDEO_INFO_FORMAT(vip), GST_VIDEO_INFO_WIDTH(vip),
+            GST_VIDEO_INFO_HEIGHT(vip), GST_VIDEO_INFO_N_PLANES(vip),
+            &GST_VIDEO_INFO_PLANE_OFFSET(vip, 0),
+            &GST_VIDEO_INFO_PLANE_STRIDE(vip, 0));
+        vmeta->map = gst_video_meta_map_vaapi_memory;
+        vmeta->unmap = gst_video_meta_unmap_vaapi_memory;
     }
 
     *out_buffer_ptr = buffer;

diff --git a/gst/vaapi/gstvaapivideomemory.c b/gst/vaapi/gstvaapivideomemory.c
index d016c12..a7d65ca 100644 (file)
--- a/gst/vaapi/gstvaapivideomemory.c
+++ b/gst/vaapi/gstvaapivideomemory.c
@@ -25,10 +25,160 @@
 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapivideomemory);
 #define GST_CAT_DEFAULT gst_debug_vaapivideomemory
 
+#ifndef GST_VIDEO_INFO_FORMAT_STRING
+#define GST_VIDEO_INFO_FORMAT_STRING(vip) \
+    gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(vip))
+#endif
+
 /* ------------------------------------------------------------------------ */
 /* --- GstVaapiVideoMemory                                              --- */
 /* ------------------------------------------------------------------------ */
 
+static GstVaapiImage *
+new_image(GstVaapiDisplay *display, const GstVideoInfo *vip)
+{
+    GstVaapiImageFormat format;
+
+    format = gst_vaapi_image_format_from_video(GST_VIDEO_INFO_FORMAT(vip));
+    if (!format)
+        return NULL;
+
+    return gst_vaapi_image_new(display, format,
+        GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+}
+
+static gboolean
+ensure_image(GstVaapiVideoMemory *mem)
+{
+    if (!mem->image) {
+        GstVaapiDisplay * const display =
+            gst_vaapi_video_meta_get_display(mem->meta);
+
+        mem->image = new_image(display, mem->image_info);
+        if (!mem->image)
+            return FALSE;
+    }
+    gst_vaapi_video_meta_set_image(mem->meta, mem->image);
+    return TRUE;
+}
+
+static GstVaapiSurface *
+new_surface(GstVaapiDisplay *display, const GstVideoInfo *vip)
+{
+    if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_NV12)
+        return NULL;
+
+    return gst_vaapi_surface_new(display, GST_VAAPI_CHROMA_TYPE_YUV420,
+        GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+}
+
+static gboolean
+ensure_surface(GstVaapiVideoMemory *mem)
+{
+    if (!mem->surface) {
+        GstVaapiDisplay * const display =
+            gst_vaapi_video_meta_get_display(mem->meta);
+
+        mem->surface = new_surface(display, mem->surface_info);
+        if (!mem->surface)
+            return FALSE;
+    }
+    gst_vaapi_video_meta_set_surface(mem->meta, mem->surface);
+    return TRUE;
+}
+
+gboolean
+gst_video_meta_map_vaapi_memory(GstVideoMeta *meta, guint plane,
+    GstMapInfo *info, gpointer *data, gint *stride, GstMapFlags flags)
+{
+    GstVaapiVideoMemory * const mem =
+        GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_get_memory(meta->buffer, 0));
+
+    g_return_val_if_fail(mem, FALSE);
+    g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
+                             allocator), FALSE);
+    g_return_val_if_fail(mem->meta, FALSE);
+
+    if ((flags & GST_MAP_READWRITE) == GST_MAP_READ)
+        goto error_unsupported_map;
+
+    if (++mem->map_count == 1) {
+        if (!ensure_surface(mem))
+            goto error_ensure_surface;
+        if (!ensure_image(mem))
+            goto error_ensure_image;
+        if (!gst_vaapi_image_map(mem->image))
+            goto error_map_image;
+    }
+
+    *data = gst_vaapi_image_get_plane(mem->image, plane);
+    *stride = gst_vaapi_image_get_pitch(mem->image, plane);
+    info->flags = flags;
+    return TRUE;
+
+    /* ERRORS */
+error_unsupported_map:
+    {
+        GST_ERROR("unsupported map flags (0x%x)", flags);
+        return FALSE;
+    }
+error_ensure_surface:
+    {
+        const GstVideoInfo * const vip = mem->surface_info;
+        GST_ERROR("failed to create %s surface of size %ux%u",
+                  GST_VIDEO_INFO_FORMAT_STRING(vip),
+                  GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+        return FALSE;
+    }
+error_ensure_image:
+    {
+        const GstVideoInfo * const vip = mem->image_info;
+        GST_ERROR("failed to create %s image of size %ux%u",
+                  GST_VIDEO_INFO_FORMAT_STRING(vip),
+                  GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+        return FALSE;
+    }
+error_map_image:
+    {
+        GST_ERROR("failed to map image %" GST_VAAPI_ID_FORMAT,
+                  GST_VAAPI_ID_ARGS(gst_vaapi_image_get_id(mem->image)));
+        return FALSE;
+    }
+}
+
+gboolean
+gst_video_meta_unmap_vaapi_memory(GstVideoMeta *meta, guint plane,
+    GstMapInfo *info)
+{
+    GstVaapiVideoMemory * const mem =
+        GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_get_memory(meta->buffer, 0));
+
+    g_return_val_if_fail(mem, FALSE);
+    g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
+                             allocator), FALSE);
+    g_return_val_if_fail(mem->meta, FALSE);
+    g_return_val_if_fail(mem->surface, FALSE);
+    g_return_val_if_fail(mem->image, FALSE);
+
+    if (--mem->map_count == 0) {
+        gst_vaapi_image_unmap(mem->image);
+
+        /* Commit VA image to surface */
+        if (info->flags & GST_MAP_WRITE) {
+            if (!gst_vaapi_surface_put_image(mem->surface, mem->image))
+                goto error_upload_image;
+        }
+    }
+    return TRUE;
+
+    /* ERRORS */
+error_upload_image:
+    {
+        GST_ERROR("failed to upload image");
+        return FALSE;
+    }
+}
+
 GstMemory *
 gst_vaapi_video_memory_new(GstAllocator *base_allocator,
     GstVaapiVideoMeta *meta)
@@ -42,17 +192,24 @@ gst_vaapi_video_memory_new(GstAllocator *base_allocator,
     if (!mem)
         return NULL;
 
-    vip = &allocator->video_info;
+    vip = &allocator->image_info;
     gst_memory_init(&mem->parent_instance, 0, base_allocator, NULL,
         GST_VIDEO_INFO_SIZE(vip), 0, 0, GST_VIDEO_INFO_SIZE(vip));
 
+    mem->surface_info = &allocator->surface_info;
+    mem->surface = NULL;
+    mem->image_info = &allocator->image_info;
+    mem->image = NULL;
     mem->meta = gst_vaapi_video_meta_ref(meta);
+    mem->map_count = 0;
     return GST_MEMORY_CAST(mem);
 }
 
 static void
 gst_vaapi_video_memory_free(GstVaapiVideoMemory *mem)
 {
+    g_clear_object(&mem->surface);
+    g_clear_object(&mem->image);
     gst_vaapi_video_meta_unref(mem->meta);
     g_slice_free(GstVaapiVideoMemory, mem);
 }
@@ -156,11 +313,48 @@ gst_vaapi_video_allocator_init(GstVaapiVideoAllocator *allocator)
         gst_vaapi_video_memory_is_span;
 }
 
+static gboolean
+gst_video_info_update_from_image(GstVideoInfo *vip, GstVaapiImage *image)
+{
+    const guchar *data;
+    guint i, num_planes, data_size;
+
+    num_planes = gst_vaapi_image_get_plane_count(image);
+    g_return_val_if_fail(num_planes == GST_VIDEO_INFO_N_PLANES(vip), FALSE);
+
+    /* Determine the base data pointer */
+    data = gst_vaapi_image_get_plane(image, 0);
+    for (i = 1; i < num_planes; i++) {
+        const guchar * const plane = gst_vaapi_image_get_plane(image, i);
+        if (data > plane)
+            data = plane;
+    }
+    data_size = gst_vaapi_image_get_data_size(image);
+
+    /* Check that we don't have disjoint planes */
+    for (i = 0; i < num_planes; i++) {
+        const guchar * const plane = gst_vaapi_image_get_plane(image, i);
+        if (plane - data > data_size)
+            return FALSE;
+    }
+
+    /* Update GstVideoInfo structure */
+    for (i = 0; i < num_planes; i++) {
+        const guchar * const plane = gst_vaapi_image_get_plane(image, i);
+        GST_VIDEO_INFO_PLANE_OFFSET(vip, i) = plane - data;
+        GST_VIDEO_INFO_PLANE_STRIDE(vip, i) =
+            gst_vaapi_image_get_pitch(image, i);
+    }
+    GST_VIDEO_INFO_SIZE(vip) = data_size;
+    return TRUE;
+}
+
 GstAllocator *
 gst_vaapi_video_allocator_new(GstVaapiDisplay *display, GstCaps *caps)
 {
     GstVaapiVideoAllocator *allocator;
     GstVideoInfo *vip;
+    GstVaapiImage *image;
 
     g_return_val_if_fail(GST_VAAPI_IS_DISPLAY(display), NULL);
     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
@@ -173,5 +367,25 @@ gst_vaapi_video_allocator_new(GstVaapiDisplay *display, GstCaps *caps)
     gst_video_info_init(vip);
     gst_video_info_from_caps(vip, caps);
 
+    gst_video_info_set_format(&allocator->surface_info, GST_VIDEO_FORMAT_NV12,
+        GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+
+    allocator->image_info = *vip;
+    if (GST_VIDEO_INFO_FORMAT(vip) == GST_VIDEO_FORMAT_ENCODED)
+        gst_video_info_set_format(&allocator->image_info, GST_VIDEO_FORMAT_NV12,
+            GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
+
+    if (1) {
+        do {
+            image = new_image(display, &allocator->image_info);
+            if (!image)
+                break;
+            if (!gst_vaapi_image_map(image))
+                break;
+            gst_video_info_update_from_image(&allocator->image_info, image);
+            gst_vaapi_image_unmap(image);
+        } while (0);
+        g_clear_object(&image);
+    }
     return GST_ALLOCATOR_CAST(allocator);
 }

diff --git a/gst/vaapi/gstvaapivideomemory.h b/gst/vaapi/gstvaapivideomemory.h
index 447dd93..29f9582 100644 (file)
--- a/gst/vaapi/gstvaapivideomemory.h
+++ b/gst/vaapi/gstvaapivideomemory.h
@@ -52,13 +52,28 @@ struct _GstVaapiVideoMemory {
     GstMemory parent_instance;
 
     /*< private >*/
+    const GstVideoInfo *surface_info;
+    GstVaapiSurface    *surface;
+    const GstVideoInfo *image_info;
+    GstVaapiImage      *image;
     GstVaapiVideoMeta  *meta;
+    gint                map_count;
 };
 
 G_GNUC_INTERNAL
 GstMemory *
 gst_vaapi_video_memory_new(GstAllocator *allocator, GstVaapiVideoMeta *meta);
 
+G_GNUC_INTERNAL
+gboolean
+gst_video_meta_map_vaapi_memory(GstVideoMeta *meta, guint plane,
+    GstMapInfo *info, gpointer *data, gint *stride, GstMapFlags flags);
+
+G_GNUC_INTERNAL
+gboolean
+gst_video_meta_unmap_vaapi_memory(GstVideoMeta *meta, guint plane,
+    GstMapInfo *info);
+
 /* ------------------------------------------------------------------------ */
 /* --- GstVaapiVideoAllocator                                           --- */
 /* ------------------------------------------------------------------------ */
@@ -89,6 +104,8 @@ struct _GstVaapiVideoAllocator {
 
     /*< private >*/
     GstVideoInfo        video_info;
+    GstVideoInfo        surface_info;
+    GstVideoInfo        image_info;
 };
 
 /**