From 63f73879864dace9533c3ffa296742c0b529f706 Mon Sep 17 00:00:00 2001 From: Sangchul Lee Date: Thu, 4 Feb 2021 16:13:05 +0900 Subject: [PATCH] webrtc_source: Add support for media packet with gstbuffer pointer In case of using media packet source, now we support two kinds of media packet. One is normal buffer which memory is allocated inside of the packet, the other one is having an external buffer without allocation. In the latter case, we assume that extra data of the media packet has a gstreamer buffer pointer. A test case for this is also added in webrtc_test.c. [Version] 0.1.114 [Issue Type] Improvement Change-Id: I391de40b6f5217e6a797e8a19d97a5640e093632 Signed-off-by: Sangchul Lee --- packaging/capi-media-webrtc.spec | 2 +- src/webrtc_source.c | 103 ++++++++++++++++++++++++------- test/webrtc_test.c | 63 +++++++++++++++++-- 3 files changed, 139 insertions(+), 29 deletions(-) diff --git a/packaging/capi-media-webrtc.spec b/packaging/capi-media-webrtc.spec index 42de0987..85a2b53e 100644 --- a/packaging/capi-media-webrtc.spec +++ b/packaging/capi-media-webrtc.spec @@ -1,6 +1,6 @@ Name: capi-media-webrtc Summary: A WebRTC library in Tizen Native API -Version: 0.1.113 +Version: 0.1.114 Release: 0 Group: Multimedia/API License: Apache-2.0 diff --git a/src/webrtc_source.c b/src/webrtc_source.c index ddac1fa7..61906dcd 100644 --- a/src/webrtc_source.c +++ b/src/webrtc_source.c @@ -18,6 +18,7 @@ #include "webrtc.h" #include "webrtc_private.h" #include +#include #define GST_KLASS_NAME_ENCODER_AUDIO "Codec/Encoder/Audio" #define GST_KLASS_NAME_ENCODER_VIDEO "Codec/Encoder/Video" @@ -1378,12 +1379,35 @@ static int __fill_gst_buffer_mapped_data_from_packet(GstBuffer *buffer, media_pa return WEBRTC_ERROR_NONE; } +static bool __is_valid_encoded_format(media_type_e type, media_format_h format) +{ + int ret; + media_format_mimetype_e mime_type; + + RET_VAL_IF(format == NULL, false, "format is NULL"); + + if (type == MEDIA_TYPE_AUDIO) { + ret = media_format_get_audio_info(format, &mime_type, NULL, NULL, NULL, NULL); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, false, "failed to media_format_get_audio_info()"); + + } else if (type == MEDIA_TYPE_VIDEO) { + ret = media_format_get_video_info(format, &mime_type, NULL, NULL, NULL, NULL); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, false, "failed to media_format_get_video_info()"); + + } else { + LOG_ERROR_IF_REACHED("type(%u)", type); + return false; + } + + return (mime_type & MEDIA_FORMAT_ENCODED); +} + int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h packet) { int ret = WEBRTC_ERROR_NONE; webrtc_gst_slot_s *source; - guchar *data_ptr; GstElement *appsrc; + GstBuffer *buffer = NULL; GstBuffer *new_buffer; guint64 pts = 0; guint64 dts = 0; @@ -1397,10 +1421,7 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h RET_VAL_IF(source->media_format == NULL, WEBRTC_ERROR_INVALID_OPERATION, "media_format is NULL"); appsrc = __find_element_in_bin(source->bin, "appsrc"); - if (appsrc == NULL) { - LOG_ERROR("source_id[%u] is not for media packet source", source_id); - return WEBRTC_ERROR_INVALID_PARAMETER; - } + RET_VAL_IF(appsrc == NULL, WEBRTC_ERROR_INVALID_PARAMETER, "source_id[%u] is not for media packet source", source_id); if (packet == NULL) { LOG_INFO("packet is NULL, emit EOS signal"); @@ -1412,43 +1433,81 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h return WEBRTC_ERROR_NONE; } - media_packet_get_buffer_data_ptr(packet, (void **)&data_ptr); - if (data_ptr == NULL) { - LOG_ERROR("invalid packet, data_ptr is NULL"); - return WEBRTC_ERROR_INVALID_OPERATION; + /* FIXME: we assume that extra field has gstreamer buffer pointer */ + ret = media_packet_get_extra(packet, (void **)&buffer); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_extra()"); + if (buffer) { + media_format_h format; + /* allow only when both media packet and appsrc are for encoded data */ + if (!__is_valid_encoded_format(source->media_types, source->media_format)) { + LOG_ERROR("media packet source is not for encoded format"); + return WEBRTC_ERROR_INVALID_OPERATION; + } + ret = media_packet_get_format(packet, &format); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_format()"); + media_format_unref(format); + if (!__is_valid_encoded_format(source->media_types, format)) { + LOG_ERROR("invalid format of packet"); + return WEBRTC_ERROR_INVALID_OPERATION; + } + + /* FIXME: need more API to check if it is valid gstreamer buffer. */ + LOG_DEBUG("external gst buffer[%p]", buffer); + g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", buffer, &gst_ret, NULL); + if (gst_ret != GST_FLOW_OK) { + LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret); + return WEBRTC_ERROR_INVALID_OPERATION; + } + return WEBRTC_ERROR_NONE; } - media_packet_get_buffer_size(packet, &size); + /* FIXME: make subfunction for codes below */ + ret = media_packet_get_buffer_size(packet, &size); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_buffer_size()"); new_buffer = gst_buffer_new_and_alloc(size); - if (!new_buffer) { - LOG_ERROR("failed to gst_buffer_new_and_alloc()"); - return WEBRTC_ERROR_INVALID_OPERATION; - } + RET_VAL_IF(new_buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "failed to gst_buffer_new_and_alloc()"); ret = __fill_gst_buffer_mapped_data_from_packet(new_buffer, packet); if (ret != WEBRTC_ERROR_NONE) { LOG_ERROR("failed to __fill_gst_buffer_mapped_data_from_packet()"); - return ret; + goto exit; } - media_packet_get_pts(packet, &pts); + ret = media_packet_get_pts(packet, &pts); + if (ret != MEDIA_PACKET_ERROR_NONE) { + LOG_ERROR("failed to media_packet_get_pts()"); + ret = WEBRTC_ERROR_INVALID_OPERATION; + goto exit; + } GST_BUFFER_PTS(new_buffer) = pts; - media_packet_get_dts(packet, &dts); + + ret = media_packet_get_dts(packet, &dts); + if (ret != MEDIA_PACKET_ERROR_NONE) { + LOG_ERROR("failed to media_packet_get_dts()"); + ret = WEBRTC_ERROR_INVALID_OPERATION; + goto exit; + } GST_BUFFER_DTS(new_buffer) = dts; - media_packet_get_duration(packet, &duration); + ret = media_packet_get_duration(packet, &duration); + if (ret != MEDIA_PACKET_ERROR_NONE) { + LOG_ERROR("failed to media_packet_get_duration()"); + ret = WEBRTC_ERROR_INVALID_OPERATION; + goto exit; + } GST_BUFFER_DURATION(new_buffer) = duration; LOG_DEBUG("new gst buffer[%p, pts:%llu, dts:%llu, duration:%llu]", new_buffer, pts, dts, duration); g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", new_buffer, &gst_ret, NULL); - gst_buffer_unref(new_buffer); - if (gst_ret != GST_FLOW_OK) { LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret); - return WEBRTC_ERROR_INVALID_OPERATION; + ret = WEBRTC_ERROR_INVALID_OPERATION; } - return WEBRTC_ERROR_NONE; +exit: + gst_buffer_unref(new_buffer); + + return ret; } diff --git a/test/webrtc_test.c b/test/webrtc_test.c index 733e4dab..aa87dd17 100644 --- a/test/webrtc_test.c +++ b/test/webrtc_test.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,8 @@ #define MAX_CONNECTION_LEN 4 #define MAX_MEDIA_PACKET_SOURCE_LEN 4 +#define USE_GSTBUFFER_WITHOUT_COPY true + enum { CURRENT_STATUS_MAINMENU, CURRENT_STATUS_ADD_MEDIA_SOURCE, @@ -1526,12 +1529,20 @@ gulong _connect_signal(GObject *obj, const char *signal_name, GCallback callback static int __media_packet_finalize_cb(media_packet_h packet, int error_code, void *user_data) { + GstBuffer *buffer = NULL; + g_print("__media_packet_finalize_cb(), packet[%p], error_code[0x%x], user_data[%p]\n", packet, error_code, user_data); + media_packet_get_extra(packet, (void **)&buffer); + if (buffer) { + g_print("unref gstbuffer[%p]\n", buffer); + gst_buffer_unref(buffer); + } + return MEDIA_PACKET_FINALIZE; } -static media_packet_h __make_media_packet(media_packet_source_s *packet_source, GstBuffer *buffer) +static media_packet_h __make_media_packet(media_packet_source_s *packet_source, GstBuffer *buffer, bool set_buffer_as_external_memory) { int ret; media_packet_h packet = NULL; @@ -1553,6 +1564,40 @@ static media_packet_h __make_media_packet(media_packet_source_s *packet_source, return NULL; } + if (set_buffer_as_external_memory) { + gst_buffer_ref(buffer); + + if (!gst_buffer_map(buffer, &buff_info, GST_MAP_READ)) { + g_printerr("failed to gst_buffer_map()\n"); + gst_buffer_unref(buffer); + return NULL; + } + + ret = media_packet_create_from_external_memory(packet_source->format, buff_info.data, gst_buffer_get_size(buffer), __media_packet_finalize_cb, packet_source, &packet); + if (ret != MEDIA_PACKET_ERROR_NONE) { + g_printerr("failed to media_packet_create_from_external_memory()\n"); + gst_buffer_unmap(buffer, &buff_info); + gst_buffer_unref(buffer); + return NULL; + } + gst_buffer_unmap(buffer, &buff_info); + + ret |= media_packet_set_pts(packet, GST_BUFFER_PTS(buffer)); + ret |= media_packet_set_dts(packet, GST_BUFFER_DTS(buffer)); + ret |= media_packet_set_duration(packet, GST_BUFFER_DURATION(buffer)); + + /* FIXME: We put the gstbuffer to extra field of media packet. It does not guarantee the validity of the gstbuffer + * after destroying this media packet. Currently, the gstbuffer must be used before destroying the packet. */ + ret |= media_packet_set_extra(packet, (void *)buffer); + if (ret != MEDIA_PACKET_ERROR_NONE) { + g_printerr("failed to media_packet_set_*()\n"); + gst_buffer_unref(buffer); + return NULL; + } + + return packet; + } + ret = media_packet_create_alloc(packet_source->format, __media_packet_finalize_cb, packet_source, &packet); if (ret != MEDIA_PACKET_ERROR_NONE) { g_printerr("failed to media_packet_create_alloc()\n"); @@ -1630,6 +1675,8 @@ static media_packet_h __make_media_packet(media_packet_source_s *packet_source, goto error; } + gst_buffer_unref(buffer); + return packet; error: @@ -1713,6 +1760,13 @@ GstBuffer *__get_buffer_from_packet(media_packet_h packet) guint64 dts = 0; guint64 duration = 0; guint64 size = 0; + GstBuffer *buffer = NULL; + + media_packet_get_extra(packet, (void **)&buffer); + if (buffer) { + g_print("external gstbuffer[%p]\n", buffer); + return buffer; + } media_packet_get_buffer_data_ptr(packet, (void **)&data_ptr); if (data_ptr == NULL) { @@ -1778,7 +1832,7 @@ static void __stream_handoff_cb(GstElement *object, GstBuffer *buffer, GstPad *p g_free(caps_str); gst_caps_unref(caps); - packet = __make_media_packet(packet_source, buffer); + packet = __make_media_packet(packet_source, buffer, USE_GSTBUFFER_WITHOUT_COPY); if (!packet) { g_printerr("failed to __make_media_packet()\n"); return; @@ -1796,9 +1850,6 @@ static void __stream_handoff_cb(GstElement *object, GstBuffer *buffer, GstPad *p g_signal_emit_by_name(G_OBJECT(packet_source->appsrc), "push-buffer", buffer_from_packet, &gst_ret, NULL); if (gst_ret != GST_FLOW_OK) g_printerr("failed to 'push-buffer', gst_ret[0x%x]\n", gst_ret); - - gst_buffer_unref(buffer_from_packet); - #else if (webrtc_media_packet_source_push_packet(packet_source->webrtc, packet_source->source_id, packet) != WEBRTC_ERROR_NONE) g_printerr("failed to webrtc_media_packet_source_push_packet()\n"); @@ -1902,7 +1953,7 @@ static void __demux_pad_added_cb(GstElement *object, GstPad *pad, gpointer data) media_type = gst_structure_get_name(gst_caps_get_structure(gst_pad_get_current_caps(pad), 0)); if (g_strrstr(media_type, "audio")) { - g_print("skip pad for audio"); + g_print("skip pad for audio\n"); return; } -- 2.34.1