From: Sangchul Lee Date: Thu, 8 Apr 2021 12:21:40 +0000 (+0900) Subject: webrtc_source: Add support for media packet with zerocopy buffer X-Git-Tag: submit/tizen/20210729.023123~95 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b59e062ecbae5a325db7bedca58e32f86fb4472f;p=platform%2Fcore%2Fapi%2Fwebrtc.git webrtc_source: Add support for media packet with zerocopy buffer If the incoming media packet has tbm surface, it should be used to allocate memory with gst_tizen_allocator_alloc_surface() to have zerocopy method. [Version] 0.1.146 [Issue Type] Improvement Change-Id: I1686f25b7140bf47331b7cd036cba0b3b4e5584e Signed-off-by: Sangchul Lee --- diff --git a/CMakeLists.txt b/CMakeLists.txt index f1d2954b..f56e71cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,8 +12,8 @@ SET(INC_DIR include) INCLUDE_DIRECTORIES(${INC_DIR}) SET(dependents "dlog glib-2.0 gstreamer-1.0 gstreamer-webrtc-1.0 gstreamer-video-1.0 gstreamer-audio-1.0 \ - json-glib-1.0 iniparser mm-common mm-display-interface capi-media-tool libtbm libwebsockets \ - cynara-client libsmack capi-system-info libsoup-2.4 bundle") + gstreamer-allocators-1.0 json-glib-1.0 iniparser mm-common mm-display-interface capi-media-tool \ + libtbm libwebsockets cynara-client libsmack capi-system-info libsoup-2.4 bundle") IF(NOT TIZEN_PROFILE_TV) SET(dependents "${dependents} mm-resource-manager") ADD_DEFINITIONS("-DTIZEN_FEATURE_RES_MGR") diff --git a/include/webrtc_private.h b/include/webrtc_private.h index 6f05e673..92bc907b 100644 --- a/include/webrtc_private.h +++ b/include/webrtc_private.h @@ -383,6 +383,7 @@ typedef struct _webrtc_gst_slot_s { int mlines[2]; /* index 0 for audio, 1 for video */ media_format_h media_format; bool zerocopy_enabled; + GstAllocator *allocator; webrtc_callbacks_s buffer_state_changed_cb; webrtc_callbacks_s *encoded_frame_cb; webrtc_s *webrtc; diff --git a/packaging/capi-media-webrtc.spec b/packaging/capi-media-webrtc.spec index 300ebd3c..9432c882 100644 --- a/packaging/capi-media-webrtc.spec +++ b/packaging/capi-media-webrtc.spec @@ -1,6 +1,6 @@ Name: capi-media-webrtc Summary: A WebRTC library in Tizen Native API -Version: 0.1.145 +Version: 0.1.146 Release: 0 Group: Multimedia/API License: Apache-2.0 @@ -16,6 +16,7 @@ BuildRequires: pkgconfig(gstreamer-1.0) BuildRequires: pkgconfig(gstreamer-webrtc-1.0) BuildRequires: pkgconfig(gstreamer-video-1.0) BuildRequires: pkgconfig(gstreamer-audio-1.0) +BuildRequires: pkgconfig(gstreamer-allocators-1.0) BuildRequires: pkgconfig(appcore-efl) BuildRequires: pkgconfig(elementary) BuildRequires: pkgconfig(json-glib-1.0) diff --git a/src/webrtc_source.c b/src/webrtc_source.c index 83a0f3eb..c9572b49 100644 --- a/src/webrtc_source.c +++ b/src/webrtc_source.c @@ -19,6 +19,7 @@ #include "webrtc_private.h" #include #include +#include #define GST_KLASS_NAME_ENCODER_AUDIO "Codec/Encoder/Audio" #define GST_KLASS_NAME_ENCODER_VIDEO "Codec/Encoder/Video" @@ -52,6 +53,11 @@ typedef struct { const int clock_rate; } payload_type_s; +typedef struct { + media_packet_h packet; + GstBuffer *buffer; +} packet_buffer_s; + static payload_type_s payload_types[] = { /* AUDIO */ [CODEC_TYPE_OPUS] = { "OPUS", 48000 }, @@ -1139,6 +1145,9 @@ void _source_slot_destroy_cb(gpointer data) if (source->media_format) media_format_unref(source->media_format); + if (source->allocator) + gst_object_unref(source->allocator); + g_free(source); } @@ -1437,13 +1446,88 @@ static bool __is_valid_format(media_type_e type, media_format_h conf_format, med return true; } +static void __memory_finalize_cb(packet_buffer_s *packet_buffer) +{ + LOG_DEBUG("packet[%p] buffer[%p] is about to release", packet_buffer->packet, packet_buffer->buffer); + + media_packet_destroy(packet_buffer->packet); + g_free(packet_buffer); +} + +static GstBuffer* __make_buffer_from_zerocopy_video_packet(webrtc_gst_slot_s *source, media_packet_h packet) +{ + int ret = MEDIA_PACKET_ERROR_NONE; + gint stride_width; + gint stride_height; + uint32_t plane_num; + guint8 *planes[2]; + guint64 pts = 0; + guint64 dts = 0; + guint64 duration = 0; + tbm_surface_h surface = NULL; + GstVideoInfo vinfo; + GstMemory *mem; + GstBuffer *buffer; + packet_buffer_s *packet_buffer; + + RET_VAL_IF(source == NULL, NULL, "source is NULL"); + RET_VAL_IF(packet == NULL, NULL, "packet is NULL"); + + ret = media_packet_get_number_of_video_planes(packet, &plane_num); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_number_of_video_planes()"); + + ret = media_packet_get_video_plane_data_ptr(packet, 0, (void **)&planes[0]); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_plane_data_ptr()"); + + ret = media_packet_get_video_stride_width(packet, 0, &stride_width); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_stride_width()"); + + ret = media_packet_get_video_stride_height(packet, 0, &stride_height); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_stride_height()"); + + ret = media_packet_get_tbm_surface(packet, &surface); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_tbm_surface()"); + + ret = media_packet_get_pts(packet, &pts); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_pts()"); + + ret = media_packet_get_dts(packet, &dts); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_dts()"); + + ret = media_packet_get_duration(packet, &duration); + RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_duration()"); + + buffer = gst_buffer_new(); + + packet_buffer = g_new0(packet_buffer_s, 1); + packet_buffer->packet = packet; + packet_buffer->buffer = buffer; + + memset(&vinfo, 0, sizeof(GstVideoInfo)); + mem = gst_tizen_allocator_alloc_surface(source->allocator, &vinfo, surface, packet_buffer, (GDestroyNotify)__memory_finalize_cb); + if (!mem) { + LOG_ERROR("failed to gst_tizen_allocator_alloc_surface()"); + gst_buffer_unref(buffer); + g_free(packet_buffer); + return NULL; + } + + GST_BUFFER_PTS(buffer) = pts; + GST_BUFFER_DTS(buffer) = dts; + GST_BUFFER_DURATION(buffer) = duration; + gst_buffer_append_memory(buffer, mem); + + LOG_DEBUG("buffer[%p, pts:%llu, dts:%llu, duration:%llu, tizen memory:%p]", buffer, pts, dts, duration, mem); + + return buffer; +} + int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h packet) { int ret = WEBRTC_ERROR_NONE; webrtc_gst_slot_s *source; GstElement *appsrc; GstBuffer *buffer = NULL; - GstBuffer *new_buffer; guint64 pts = 0; guint64 dts = 0; guint64 duration = 0; @@ -1468,6 +1552,23 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h return WEBRTC_ERROR_NONE; } + /* the incoming media packet should have zerocopy format (e.g., SN12) */ + if (source->zerocopy_enabled) { + if (!source->allocator) + source->allocator = gst_tizen_allocator_new(); + buffer = __make_buffer_from_zerocopy_video_packet(source, packet); + RET_VAL_IF(buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "buffer is NULL"); + + g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", buffer, &gst_ret, NULL); + if (gst_ret != GST_FLOW_OK) { + LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret); + gst_buffer_unref(buffer); + return WEBRTC_ERROR_INVALID_OPERATION; + } + gst_buffer_unref(buffer); + return WEBRTC_ERROR_NONE; + } + /* FIXME: we assume that extra field has gstreamer buffer pointer */ ret = media_packet_get_extra(packet, (void **)&buffer); RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_extra()"); @@ -1497,10 +1598,10 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h ret = media_packet_get_buffer_size(packet, &size); RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_buffer_size()"); - new_buffer = gst_buffer_new_and_alloc(size); - RET_VAL_IF(new_buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "failed to gst_buffer_new_and_alloc()"); + buffer = gst_buffer_new_and_alloc(size); + RET_VAL_IF(buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "failed to gst_buffer_new_and_alloc()"); - ret = __fill_gst_buffer_mapped_data_from_packet(new_buffer, packet); + ret = __fill_gst_buffer_mapped_data_from_packet(buffer, packet); if (ret != WEBRTC_ERROR_NONE) { LOG_ERROR("failed to __fill_gst_buffer_mapped_data_from_packet()"); goto exit; @@ -1512,7 +1613,7 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h ret = WEBRTC_ERROR_INVALID_OPERATION; goto exit; } - GST_BUFFER_PTS(new_buffer) = pts; + GST_BUFFER_PTS(buffer) = pts; ret = media_packet_get_dts(packet, &dts); if (ret != MEDIA_PACKET_ERROR_NONE) { @@ -1520,7 +1621,7 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h ret = WEBRTC_ERROR_INVALID_OPERATION; goto exit; } - GST_BUFFER_DTS(new_buffer) = dts; + GST_BUFFER_DTS(buffer) = dts; ret = media_packet_get_duration(packet, &duration); if (ret != MEDIA_PACKET_ERROR_NONE) { @@ -1528,18 +1629,18 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h ret = WEBRTC_ERROR_INVALID_OPERATION; goto exit; } - GST_BUFFER_DURATION(new_buffer) = duration; + GST_BUFFER_DURATION(buffer) = duration; - LOG_DEBUG("new gst buffer[%p, pts:%llu, dts:%llu, duration:%llu]", new_buffer, pts, dts, duration); + LOG_DEBUG("new gst buffer[%p, pts:%llu, dts:%llu, duration:%llu]", buffer, pts, dts, duration); - g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", new_buffer, &gst_ret, NULL); + g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", buffer, &gst_ret, NULL); if (gst_ret != GST_FLOW_OK) { LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret); ret = WEBRTC_ERROR_INVALID_OPERATION; } exit: - gst_buffer_unref(new_buffer); + gst_buffer_unref(buffer); if (ret == WEBRTC_ERROR_NONE) media_packet_destroy(packet);