Add video meta for video encoder

Some encoder plugin needs video meta from GST buffer,
but GST buffer from media codec did not provide video meta.
This patch sets video meta to GST buffer.

- Changes
 : Change return type for mc_caps_new()(GstCaps* to gboolean).
 : Add new functions for video info and meta.
 : Remove gst_caps_unref() in mc_gst_prepare().
   gst_caps_unref() can be called although it's freed after pipeline is released all.

[Version] 0.6.32
[Issue Type] Improvement

Change-Id: Iefb2e17bee72a3e7a72bdf952f8c0fa2e2f98dd4
Signed-off-by: Jeongmo Yang <jm80.yang@samsung.com>

diff --git a/include/media_codec_port_gst.h b/include/media_codec_port_gst.h
index 8b086e6..3f63bc2 100644 (file)
--- a/include/media_codec_port_gst.h
+++ b/include/media_codec_port_gst.h
@@ -141,11 +141,13 @@ struct _mc_gst_core_t {
        GQueue *output_queue;
 
        GstBuffer *codec_data;
-       GstCaps* (*mc_caps_new)(mc_gst_core_t *, mediacodec_codec_type_e, gint);
+       gboolean (*mc_caps_new)(mc_gst_core_t *, mediacodec_codec_type_e, gint);
        GstAllocator *allocator;
 
        void* user_cb[_MEDIACODEC_EVENT_TYPE_NUM];
        void* user_data[_MEDIACODEC_EVENT_TYPE_NUM];
+
+       GstVideoInfo *video_info;
 };
 
 struct _GstMCBuffer {

diff --git a/packaging/capi-media-codec.spec b/packaging/capi-media-codec.spec
index e45275d..db7b1f7 100644 (file)
--- a/packaging/capi-media-codec.spec
+++ b/packaging/capi-media-codec.spec
@@ -4,7 +4,7 @@
 
 Name:       capi-media-codec
 Summary:    A Media Codec library in Tizen Native API
-Version:    0.6.31
+Version:    0.6.32
 Release:    0
 Group:      Multimedia/API
 License:    Apache-2.0

diff --git a/src/media_codec_port_gst.c b/src/media_codec_port_gst.c
index cf46748..2785565 100644 (file)
--- a/src/media_codec_port_gst.c
+++ b/src/media_codec_port_gst.c
@@ -60,6 +60,9 @@ static int __mc_set_caps_codecdata(mc_gst_core_t *core, GstMCBuffer *mcbuffer, g
 static void __mc_push_output_to_queue(mc_gst_core_t *core, GstMCBuffer *mcbuffer);
 static int __mc_gst_create_eos_packet(media_format_h fmt, GstMCBuffer *mcbuffer);
 static void _mc_gst_handle_input_buffer_used(mc_gst_core_t *core, media_packet_h packet);
+static gboolean _mc_gst_video_info_check_and_new(mc_gst_core_t *core, GstMCBuffer *mcbuffer);
+static void _mc_gst_video_info_free(mc_gst_core_t *core);
+static void _mc_gst_video_meta_add(mc_gst_core_t *core, GstBuffer *buffer);
 
 static gint __gst_handle_stream_error(mc_gst_core_t *core, GError *error, GstMessage *message);
 static gint __gst_transform_gsterror(mc_gst_core_t *core, GstMessage *message, GError *error);
@@ -85,8 +88,8 @@ static void _mc_send_eos_signal(mc_gst_core_t *core);
 static void _mc_wait_for_eos(mc_gst_core_t *core);
 static int _mc_get_mime(mc_gst_core_t *core);
 static int _mc_set_codec_data(mc_gst_core_t *core, GstMCBuffer *mcbuffer, bool codec_config);
-GstCaps *_mc_gst_vid_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index);
-GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index);
+static gboolean _mc_gst_vid_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index);
+static gboolean _mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index);
 
 static int __mc_fill_input_buffer(mc_gst_core_t *core, media_packet_h packet, GstMCBuffer *mcbuffer);
 static int __mc_fill_output_buffer(mc_gst_core_t *core, void *data, int size, GstMCBuffer *mcbuffer);
@@ -406,6 +409,8 @@ static int __mc_fill_input_buffer_with_venc_packet(mc_gst_core_t *core, media_pa
 
        gst_buffer_append_memory(mcbuffer->buffer, mem);
 
+       _mc_gst_video_meta_add(core, mcbuffer->buffer);
+
        return ret;
 }
 
@@ -1150,6 +1155,70 @@ gboolean _mc_update_packet_info(mc_gst_core_t *core, media_format_h format)
        return is_format_change;
 }
 
+static gboolean _mc_gst_video_info_check_and_new(mc_gst_core_t *core, GstMCBuffer *mcbuffer)
+{
+       int i = 0;
+       gchar *caps_str = NULL;
+
+       if (!core || !mcbuffer) {
+               LOGE("NULL core[%p] or mcbuffer[%p]", core, mcbuffer);
+               return FALSE;
+       }
+
+       if (!core->video_info) {
+               caps_str = gst_caps_to_string(core->caps);
+
+               LOGI("caps[%s] to video info", caps_str);
+
+               g_free(caps_str);
+
+               core->video_info = gst_video_info_new_from_caps(core->caps);
+               if (!core->video_info) {
+                       LOGE("video info failed");
+                       return FALSE;
+               }
+
+               for (i = 0 ; i < GST_VIDEO_INFO_N_PLANES(core->video_info) ; i++) {
+                       GST_VIDEO_INFO_PLANE_OFFSET(core->video_info, i) = 0;
+                       media_packet_get_video_stride_width(mcbuffer->packet, i, &GST_VIDEO_INFO_PLANE_STRIDE(core->video_info, i));
+
+                       LOGI("[%d] offset[%zu], stride[%d]", i,
+                               GST_VIDEO_INFO_PLANE_OFFSET(core->video_info, i),
+                               GST_VIDEO_INFO_PLANE_STRIDE(core->video_info, i));
+               }
+
+               LOGI("new video info[%p]", core->video_info);
+       }
+
+       return TRUE;
+}
+
+static void _mc_gst_video_info_free(mc_gst_core_t *core)
+{
+       if (!core)
+               return;
+
+       if (core->video_info) {
+               LOGI("free video info[%p]", core->video_info);
+               gst_video_info_free(core->video_info);
+               core->video_info = NULL;
+       }
+}
+
+static void _mc_gst_video_meta_add(mc_gst_core_t *core, GstBuffer *buffer)
+{
+       if (!core || !core->video_info || !buffer)
+               return;
+
+       gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
+               GST_VIDEO_INFO_FORMAT(core->video_info),
+               GST_VIDEO_INFO_WIDTH(core->video_info),
+               GST_VIDEO_INFO_HEIGHT(core->video_info),
+               GST_VIDEO_INFO_N_PLANES(core->video_info),
+               core->video_info->offset,
+               core->video_info->stride);
+}
+
 static int _mc_set_codec_data(mc_gst_core_t *core, GstMCBuffer *mcbuffer, bool codec_config)
 {
        MEDIACODEC_FENTER();
@@ -1283,12 +1352,12 @@ static gpointer feed_task(gpointer data)
                }
 
                if (is_format_change) {
-                       GstCaps *caps = core->mc_caps_new(core, core->codec_id, in_port_index);
-
-                       g_object_set(core->appsrc, "caps", caps, NULL);
-                       gst_caps_unref(caps);
+                       if (!core->mc_caps_new(core, core->codec_id, in_port_index)) {
+                               LOGE("caps new failed");
+                               goto ERROR;
+                       }
 
-                       LOGD("caps updated");
+                       g_object_set(core->appsrc, "caps", core->caps, NULL);
 
                        if (core->encoder) {
                                int bitrate = core->video ? \
@@ -1385,9 +1454,10 @@ static void __mc_gst_start_feed(GstElement *pipeline, guint size, gpointer data)
        }
 }
 
-GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index)
+static gboolean _mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index)
 {
-       GstCaps *caps = NULL;
+       gchar *caps_string = NULL;
+       g_autoptr(GstCaps) caps = NULL;
        mc_gst_port_def_t *port_def = &core->ports[index]->port_def;
 
        caps = gst_caps_new_simple(core->mime,
@@ -1414,7 +1484,7 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
        case MEDIACODEC_AAC_HE_PS:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] he-aac-v12 encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                        gst_caps_set_simple(caps,
                                "mpegversion", G_TYPE_INT, 4,
@@ -1427,7 +1497,7 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
        case MEDIACODEC_MP3:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] mp3 encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                        gst_caps_set_simple(caps,
                                "framed", G_TYPE_BOOLEAN, TRUE,
@@ -1454,7 +1524,7 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
        case MEDIACODEC_AMR_WB:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] amr-wb encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                        gst_caps_set_simple(caps,
                                "rate", G_TYPE_INT, 16000,
@@ -1464,14 +1534,14 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
        case MEDIACODEC_VORBIS:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] vorbis encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                }
                break;
        case MEDIACODEC_FLAC:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] flac encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                        gst_caps_set_simple(caps,
                                "framed", G_TYPE_BOOLEAN, TRUE,
@@ -1485,7 +1555,7 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
        case MEDIACODEC_WMALSL:
                if (core->encoder) {
                        LOGD("[MC_NOT_SUPPORTED] wma encoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                } else {
                        /*
                         * Need to extract from Stream Type Specific ... or
@@ -1530,16 +1600,30 @@ GstCaps *_mc_gst_aud_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
                                "layout", G_TYPE_STRING, "interleaved", NULL);
                } else {
                        LOGD("[MC_NOT_SUPPORTED] opus decoder is not supported yet!!!");
-                       return NULL;
+                       return FALSE;
                }
                break;
        default:
                break;
        }
-       return caps;
+
+       caps_string = gst_caps_to_string(caps);
+       if (caps_string) {
+               LOGI("new caps[%p:%s]", caps, caps_string);
+               g_free(caps_string);
+       }
+
+       if (core->caps)
+               gst_caps_unref(core->caps);
+
+       _mc_gst_video_info_free(core);
+
+       core->caps = g_steal_pointer(&caps);
+
+       return TRUE;
 }
 
-GstCaps *_mc_gst_vid_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index)
+static gboolean _mc_gst_vid_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec_id, gint index)
 {
        gchar *caps_string = NULL;
        GstCaps *caps = NULL;
@@ -1600,11 +1684,17 @@ GstCaps *_mc_gst_vid_caps_new(mc_gst_core_t *core, mediacodec_codec_type_e codec
 
        caps_string = gst_caps_to_string(caps);
        if (caps_string) {
-               LOGI("new caps[%s]", caps_string);
+               LOGI("new caps[%p:%s]", caps, caps_string);
                g_free(caps_string);
        }
 
-       return caps;
+       if (core->caps)
+               gst_caps_unref(core->caps);
+       _mc_gst_video_info_free(core);
+
+       core->caps = caps;
+
+       return TRUE;
 }
 
 static int _mc_link_vtable(mc_gst_core_t *core, mediacodec_codec_type_e id, gboolean encoder, gboolean is_hw)
@@ -1746,7 +1836,6 @@ mc_ret_e mc_gst_prepare(mc_handle_t *mc_handle)
        bool encoder;
        bool hardware;
        gchar *factory_name = NULL;
-       GstCaps *caps = NULL;
        media_format_mimetype_e out_mime;
        mediacodec_codec_type_e id;
        mc_codec_map_t *codec_map;
@@ -1829,20 +1918,19 @@ mc_ret_e mc_gst_prepare(mc_handle_t *mc_handle)
        }
 
        /* set caps in capsfilter as default*/
-       caps = new_core->mc_caps_new(new_core, id, out_port_index);
+       if (!new_core->mc_caps_new(new_core, id, out_port_index)) {
+               LOGE("caps new failed");
+               return MC_INTERNAL_ERROR;
+       }
 
-       new_core->caps = caps;
        /* FIXME will parse input format from ini. format is needed when linking elements*/
        if (new_core->video && new_core->encoder) {
-               gchar *format = NULL;
-
-               format = new_core->is_hw ? "SN12" : "I420";
-               gst_caps_set_simple(caps,
-                       "format", G_TYPE_STRING, format,
+               gst_caps_set_simple(new_core->caps,
+                       "format", G_TYPE_STRING, (new_core->is_hw ? "SN12" : "I420"),
                        NULL);
        }
-       g_object_set(new_core->capsfilter, "caps", caps, NULL);
-       gst_caps_unref(caps);
+
+       g_object_set(new_core->capsfilter, "caps", new_core->caps, NULL);
 
        _mc_gst_feed_task_new(new_core);
 
@@ -1880,6 +1968,8 @@ mc_ret_e mc_gst_unprepare(mc_handle_t *mc_handle)
 
                _mc_gst_set_flush_input(core);
 
+               _mc_gst_video_info_free(core);
+
                ret = _mc_gst_destroy_pipeline(core);
 
                /* unset callback */
@@ -2155,14 +2245,14 @@ mc_ret_e _mc_gst_create_pipeline(mc_gst_core_t *core, const gchar *factory_name)
                        core->audioconvert = gst_element_factory_make("audioconvert", NULL);
 
                        if (!core->audioconvert) {
-                               LOGE("capsfilter can't create");
+                               LOGE("audioconvert can't create");
                                goto ERROR;
                        }
 
                        core->audioresample = gst_element_factory_make("audioresample", NULL);
 
                        if (!core->audioresample) {
-                               LOGE("capsfilter can't create");
+                               LOGE("audioresample can't create");
                                goto ERROR;
                        }
                }
@@ -2340,6 +2430,7 @@ mc_ret_e _mc_gst_destroy_pipeline(mc_gst_core_t *core)
                        MEDIACODEC_ELEMENT_SET_STATE(core->pipeline, GST_STATE_NULL);
 
                        gst_object_unref(GST_OBJECT(core->pipeline));
+
                        if (core->caps) {
                                gst_caps_unref(core->caps);
                                core->caps = NULL;
@@ -2776,7 +2867,6 @@ static GstMemory *__mc_gst_make_tbm_buffer(mc_gst_core_t *core, GstMCBuffer *mcb
 {
        int ret = MEDIA_PACKET_ERROR_NONE;
        tbm_surface_h surface = NULL;
-       GstVideoInfo vinfo;
        GstMemory *mem = NULL;
 
        if (!mcbuffer->packet) {
@@ -2784,19 +2874,19 @@ static GstMemory *__mc_gst_make_tbm_buffer(mc_gst_core_t *core, GstMCBuffer *mcb
                return NULL;
        }
 
-       ret = media_packet_get_tbm_surface(mcbuffer->packet, &surface);
-       if (ret != MEDIA_PACKET_ERROR_NONE) {
-               LOGE("Failed to get tbm surface");
+       if (!_mc_gst_video_info_check_and_new(core, mcbuffer)) {
+               LOGE("video info failed");
                return NULL;
        }
 
-       if (!gst_video_info_from_caps(&vinfo, core->caps)) {
-               LOGE("Failed to get video info");
+       ret = media_packet_get_tbm_surface(mcbuffer->packet, &surface);
+       if (ret != MEDIA_PACKET_ERROR_NONE) {
+               LOGE("Failed to get tbm surface");
                return NULL;
        }
 
        mem = gst_tizen_allocator_alloc_surface(core->allocator,
-                       &vinfo, surface, mcbuffer, (GDestroyNotify)__mc_input_buffer_finalize_cb);
+                       core->video_info, surface, mcbuffer, (GDestroyNotify)__mc_input_buffer_finalize_cb);
 
        return mem;
 }