webrtc_source: Add support for media packet with zerocopy buffer

If the incoming media packet has tbm surface, it should be used to
allocate memory with gst_tizen_allocator_alloc_surface() to have
zerocopy method.

[Version] 0.1.146
[Issue Type] Improvement

Change-Id: I1686f25b7140bf47331b7cd036cba0b3b4e5584e
Signed-off-by: Sangchul Lee <sc11.lee@samsung.com>

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f1d2954bf63fa7106695cdbc94c7f14c3e0a007e..f56e71cf22d57323ec5fd6aa169047cadcb09a46 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,8 +12,8 @@ SET(INC_DIR include)
 INCLUDE_DIRECTORIES(${INC_DIR})
 
 SET(dependents "dlog glib-2.0 gstreamer-1.0 gstreamer-webrtc-1.0 gstreamer-video-1.0 gstreamer-audio-1.0 \
-                json-glib-1.0 iniparser mm-common mm-display-interface capi-media-tool libtbm libwebsockets \
-                cynara-client libsmack capi-system-info libsoup-2.4 bundle")
+                gstreamer-allocators-1.0 json-glib-1.0 iniparser mm-common mm-display-interface capi-media-tool \
+                libtbm libwebsockets cynara-client libsmack capi-system-info libsoup-2.4 bundle")
 IF(NOT TIZEN_PROFILE_TV)
     SET(dependents "${dependents} mm-resource-manager")
     ADD_DEFINITIONS("-DTIZEN_FEATURE_RES_MGR")

diff --git a/include/webrtc_private.h b/include/webrtc_private.h
index 6f05e67345a3e22966ffe586c2abdb14c83d5da4..92bc907b9b8bb9c3ba185e053015a209dc6bae2c 100644 (file)
--- a/include/webrtc_private.h
+++ b/include/webrtc_private.h
@@ -383,6 +383,7 @@ typedef struct _webrtc_gst_slot_s {
        int mlines[2];      /* index 0 for audio, 1 for video */
        media_format_h media_format;
        bool zerocopy_enabled;
+       GstAllocator *allocator;
        webrtc_callbacks_s buffer_state_changed_cb;
        webrtc_callbacks_s *encoded_frame_cb;
        webrtc_s *webrtc;

diff --git a/packaging/capi-media-webrtc.spec b/packaging/capi-media-webrtc.spec
index 300ebd3c1b625d61ac996d5858b6f1106c0283a1..9432c8826a07ef966e0705b2add2a42f34d69f83 100644 (file)
--- a/packaging/capi-media-webrtc.spec
+++ b/packaging/capi-media-webrtc.spec
@@ -1,6 +1,6 @@
 Name:       capi-media-webrtc
 Summary:    A WebRTC library in Tizen Native API
-Version:    0.1.145
+Version:    0.1.146
 Release:    0
 Group:      Multimedia/API
 License:    Apache-2.0
@@ -16,6 +16,7 @@ BuildRequires:  pkgconfig(gstreamer-1.0)
 BuildRequires:  pkgconfig(gstreamer-webrtc-1.0)
 BuildRequires:  pkgconfig(gstreamer-video-1.0)
 BuildRequires:  pkgconfig(gstreamer-audio-1.0)
+BuildRequires:  pkgconfig(gstreamer-allocators-1.0)
 BuildRequires:  pkgconfig(appcore-efl)
 BuildRequires:  pkgconfig(elementary)
 BuildRequires:  pkgconfig(json-glib-1.0)

diff --git a/src/webrtc_source.c b/src/webrtc_source.c
index 83a0f3eb81ec3a40357240f763a6c8d0dab522ea..c9572b491462c30118c61aca18037275accec097 100644 (file)
--- a/src/webrtc_source.c
+++ b/src/webrtc_source.c
@@ -19,6 +19,7 @@
 #include "webrtc_private.h"
 #include <tbm_surface_internal.h>
 #include <media_packet_internal.h>
+#include <gst/allocators/gsttizenmemory.h>
 
 #define GST_KLASS_NAME_ENCODER_AUDIO   "Codec/Encoder/Audio"
 #define GST_KLASS_NAME_ENCODER_VIDEO   "Codec/Encoder/Video"
@@ -52,6 +53,11 @@ typedef struct {
        const int clock_rate;
 } payload_type_s;
 
+typedef struct {
+       media_packet_h packet;
+       GstBuffer *buffer;
+} packet_buffer_s;
+
 static payload_type_s payload_types[] = {
        /* AUDIO */
        [CODEC_TYPE_OPUS] = { "OPUS", 48000 },
@@ -1139,6 +1145,9 @@ void _source_slot_destroy_cb(gpointer data)
        if (source->media_format)
                media_format_unref(source->media_format);
 
+       if (source->allocator)
+               gst_object_unref(source->allocator);
+
        g_free(source);
 }
 
@@ -1437,13 +1446,88 @@ static bool __is_valid_format(media_type_e type, media_format_h conf_format, med
        return true;
 }
 
+static void __memory_finalize_cb(packet_buffer_s *packet_buffer)
+{
+       LOG_DEBUG("packet[%p] buffer[%p] is about to release", packet_buffer->packet, packet_buffer->buffer);
+
+       media_packet_destroy(packet_buffer->packet);
+       g_free(packet_buffer);
+}
+
+static GstBuffer* __make_buffer_from_zerocopy_video_packet(webrtc_gst_slot_s *source, media_packet_h packet)
+{
+       int ret = MEDIA_PACKET_ERROR_NONE;
+       gint stride_width;
+       gint stride_height;
+       uint32_t plane_num;
+       guint8 *planes[2];
+       guint64 pts = 0;
+       guint64 dts = 0;
+       guint64 duration = 0;
+       tbm_surface_h surface = NULL;
+       GstVideoInfo vinfo;
+       GstMemory *mem;
+       GstBuffer *buffer;
+       packet_buffer_s *packet_buffer;
+
+       RET_VAL_IF(source == NULL, NULL, "source is NULL");
+       RET_VAL_IF(packet == NULL, NULL, "packet is NULL");
+
+       ret = media_packet_get_number_of_video_planes(packet, &plane_num);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_number_of_video_planes()");
+
+       ret = media_packet_get_video_plane_data_ptr(packet, 0, (void **)&planes[0]);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_plane_data_ptr()");
+
+       ret = media_packet_get_video_stride_width(packet, 0, &stride_width);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_stride_width()");
+
+       ret = media_packet_get_video_stride_height(packet, 0, &stride_height);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_video_stride_height()");
+
+       ret = media_packet_get_tbm_surface(packet, &surface);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_tbm_surface()");
+
+       ret = media_packet_get_pts(packet, &pts);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_pts()");
+
+       ret = media_packet_get_dts(packet, &dts);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_dts()");
+
+       ret = media_packet_get_duration(packet, &duration);
+       RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, NULL, "failed to media_packet_get_duration()");
+
+       buffer = gst_buffer_new();
+
+       packet_buffer = g_new0(packet_buffer_s, 1);
+       packet_buffer->packet = packet;
+       packet_buffer->buffer = buffer;
+
+       memset(&vinfo, 0, sizeof(GstVideoInfo));
+       mem = gst_tizen_allocator_alloc_surface(source->allocator, &vinfo, surface, packet_buffer, (GDestroyNotify)__memory_finalize_cb);
+       if (!mem) {
+               LOG_ERROR("failed to gst_tizen_allocator_alloc_surface()");
+               gst_buffer_unref(buffer);
+               g_free(packet_buffer);
+               return NULL;
+       }
+
+       GST_BUFFER_PTS(buffer) = pts;
+       GST_BUFFER_DTS(buffer) = dts;
+       GST_BUFFER_DURATION(buffer) = duration;
+       gst_buffer_append_memory(buffer, mem);
+
+       LOG_DEBUG("buffer[%p, pts:%llu, dts:%llu, duration:%llu, tizen memory:%p]", buffer, pts, dts, duration, mem);
+
+       return buffer;
+}
+
 int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h packet)
 {
        int ret = WEBRTC_ERROR_NONE;
        webrtc_gst_slot_s *source;
        GstElement *appsrc;
        GstBuffer *buffer = NULL;
-       GstBuffer *new_buffer;
        guint64 pts = 0;
        guint64 dts = 0;
        guint64 duration = 0;
@@ -1468,6 +1552,23 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h
                return WEBRTC_ERROR_NONE;
        }
 
+       /* the incoming media packet should have zerocopy format (e.g., SN12) */
+       if (source->zerocopy_enabled) {
+               if (!source->allocator)
+                       source->allocator = gst_tizen_allocator_new();
+               buffer = __make_buffer_from_zerocopy_video_packet(source, packet);
+               RET_VAL_IF(buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "buffer is NULL");
+
+               g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", buffer, &gst_ret, NULL);
+               if (gst_ret != GST_FLOW_OK) {
+                       LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret);
+                       gst_buffer_unref(buffer);
+                       return WEBRTC_ERROR_INVALID_OPERATION;
+               }
+               gst_buffer_unref(buffer);
+               return WEBRTC_ERROR_NONE;
+       }
+
        /* FIXME: we assume that extra field has gstreamer buffer pointer */
        ret = media_packet_get_extra(packet, (void **)&buffer);
        RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_extra()");
@@ -1497,10 +1598,10 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h
        ret = media_packet_get_buffer_size(packet, &size);
        RET_VAL_IF(ret != MEDIA_PACKET_ERROR_NONE, WEBRTC_ERROR_INVALID_OPERATION, "failed to media_packet_get_buffer_size()");
 
-       new_buffer = gst_buffer_new_and_alloc(size);
-       RET_VAL_IF(new_buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "failed to gst_buffer_new_and_alloc()");
+       buffer = gst_buffer_new_and_alloc(size);
+       RET_VAL_IF(buffer == NULL, WEBRTC_ERROR_INVALID_OPERATION, "failed to gst_buffer_new_and_alloc()");
 
-       ret = __fill_gst_buffer_mapped_data_from_packet(new_buffer, packet);
+       ret = __fill_gst_buffer_mapped_data_from_packet(buffer, packet);
        if (ret != WEBRTC_ERROR_NONE) {
                LOG_ERROR("failed to __fill_gst_buffer_mapped_data_from_packet()");
                goto exit;
@@ -1512,7 +1613,7 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h
                ret = WEBRTC_ERROR_INVALID_OPERATION;
                goto exit;
        }
-       GST_BUFFER_PTS(new_buffer) = pts;
+       GST_BUFFER_PTS(buffer) = pts;
 
        ret = media_packet_get_dts(packet, &dts);
        if (ret != MEDIA_PACKET_ERROR_NONE) {
@@ -1520,7 +1621,7 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h
                ret = WEBRTC_ERROR_INVALID_OPERATION;
                goto exit;
        }
-       GST_BUFFER_DTS(new_buffer) = dts;
+       GST_BUFFER_DTS(buffer) = dts;
 
        ret = media_packet_get_duration(packet, &duration);
        if (ret != MEDIA_PACKET_ERROR_NONE) {
@@ -1528,18 +1629,18 @@ int _push_media_packet(webrtc_s *webrtc, unsigned int source_id, media_packet_h
                ret = WEBRTC_ERROR_INVALID_OPERATION;
                goto exit;
        }
-       GST_BUFFER_DURATION(new_buffer) = duration;
+       GST_BUFFER_DURATION(buffer) = duration;
 
-       LOG_DEBUG("new gst buffer[%p, pts:%llu, dts:%llu, duration:%llu]", new_buffer, pts, dts, duration);
+       LOG_DEBUG("new gst buffer[%p, pts:%llu, dts:%llu, duration:%llu]", buffer, pts, dts, duration);
 
-       g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", new_buffer, &gst_ret, NULL);
+       g_signal_emit_by_name(G_OBJECT(appsrc), "push-buffer", buffer, &gst_ret, NULL);
        if (gst_ret != GST_FLOW_OK) {
                LOG_ERROR("failed to 'push-buffer', gst_ret[0x%x]", gst_ret);
                ret = WEBRTC_ERROR_INVALID_OPERATION;
        }
 
 exit:
-       gst_buffer_unref(new_buffer);
+       gst_buffer_unref(buffer);
        if (ret == WEBRTC_ERROR_NONE)
                media_packet_destroy(packet);