webrtc_test: esplusplayer integration

The esplusplayer will be activated to render received data
if an encoded frame callback is set. Use commands below.

 'sa'. Set encoded audio frame callback
 'sv'. Set encoded video frame callback

[Version] 0.3.136
[Issue Type] New feature

Change-Id: I1400be4a77b6b99db44788dcf428a0e969e9571f
Signed-off-by: Sangchul Lee <sc11.lee@samsung.com>

diff --git a/packaging/capi-media-webrtc.spec b/packaging/capi-media-webrtc.spec
index 21315a2fe83d2997486e1bdce2da40e675a8aa5e..f61a93e23b7b6895d65e67a87b1b2384418bea81 100644 (file)
--- a/packaging/capi-media-webrtc.spec
+++ b/packaging/capi-media-webrtc.spec
@@ -1,6 +1,6 @@
 Name:       capi-media-webrtc
 Summary:    A WebRTC library in Tizen Native API
-Version:    0.3.135
+Version:    0.3.136
 Release:    0
 Group:      Multimedia/API
 License:    Apache-2.0

diff --git a/test/webrtc_test.c b/test/webrtc_test.c
index 8f5b61746c431369bd1b1edd51c130e800a95cbb..5b8afc4a9dec13546326b9d192b0f2cfa4e38eb8 100644 (file)
--- a/test/webrtc_test.c
+++ b/test/webrtc_test.c
@@ -374,6 +374,138 @@ static void __release_packet_source(int conn_idx, unsigned int source_idx)
        g_cond_clear(&packet_source->cond);
 }
 
+#ifndef TIZEN_TV
+static void __espp_prepare_async_done_cb(bool result, void *user_data)
+{
+       connection_s *conn = (connection_s *)user_data;
+
+       g_print("__espp_prepare_async_done_cb() is called, result[%u] conn[%p]\n", result, conn);
+
+       g_cond_signal(&conn->render.espp.cond);
+}
+
+static void __espp_ready_to_prepare_cb(esplusplayer_stream_type type, void *user_data)
+{
+       connection_s *conn = (connection_s *)user_data;
+
+       g_print("__espp_ready_to_prepare_cb() is called, type[%u], conn[%p]\n", type, conn);
+
+       g_cond_signal(&conn->render.espp.cond);
+}
+
+static void __espp_init(int index)
+{
+       if (g_ad.conns[index].render.espp.handle)
+               return;
+
+       g_mutex_init(&g_ad.conns[index].render.espp.mutex);
+       g_cond_init(&g_ad.conns[index].render.espp.cond);
+       g_ad.conns[index].render.espp.audio_track_preparing = false;
+       g_ad.conns[index].render.espp.video_track_preparing = false;
+
+       g_print("espp create & open\n");
+       g_ad.conns[index].render.espp.handle = esplusplayer_create();
+       esplusplayer_set_prepare_async_done_cb(g_ad.conns[index].render.espp.handle, __espp_prepare_async_done_cb, &g_ad.conns[index]);
+       esplusplayer_set_ready_to_prepare_cb(g_ad.conns[index].render.espp.handle, __espp_ready_to_prepare_cb, &g_ad.conns[index]);
+       esplusplayer_open(g_ad.conns[index].render.espp.handle);
+}
+
+static void __espp_deinit(int index)
+{
+       if (!g_ad.conns[index].render.espp.handle)
+               return;
+
+       g_print("espp stop & close & destroy\n");
+       esplusplayer_stop(g_ad.conns[index].render.espp.handle);
+       esplusplayer_close(g_ad.conns[index].render.espp.handle);
+       esplusplayer_destroy(g_ad.conns[index].render.espp.handle);
+
+       g_ad.conns[index].render.espp.handle = NULL;
+       g_ad.conns[index].render.espp.audio_track_preparing = false;
+       g_ad.conns[index].render.espp.video_track_preparing = false;
+       g_mutex_clear(&g_ad.conns[index].render.espp.mutex);
+       g_cond_clear(&g_ad.conns[index].render.espp.cond);
+}
+
+static int __convert_media_format_mime_to_espp_mime(media_format_mimetype_e type)
+{
+       g_print("media_format_mimetype(0x%x)\n", type);
+
+       switch (type) {
+       case MEDIA_FORMAT_OPUS:
+               return (int)ESPLUSPLAYER_AUDIO_MIME_TYPE_OPUS;
+
+       case MEDIA_FORMAT_VP8:
+               return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_VP8;
+
+       case MEDIA_FORMAT_VP9:
+               return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_VP9;
+
+       case MEDIA_FORMAT_H264_SP:
+       case MEDIA_FORMAT_H264_MP:
+       case MEDIA_FORMAT_H264_HP:
+               return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_H264;
+
+       case MEDIA_FORMAT_MJPEG:
+               return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_MJPEG;
+
+       default:
+               return -1;
+       }
+}
+
+static void __espp_update_audio_info(connection_s *conn, media_packet_h packet)
+{
+       media_format_h format;
+       media_format_mimetype_e mimetype;
+       int type;
+       int channels = 0;
+       int rate = 0;
+
+       RET_IF(!conn, "conn is NULL");
+       RET_IF(!packet, "packet is NULL");
+       RET_IF(media_packet_get_format(packet, &format) != MEDIA_PACKET_ERROR_NONE, "format is NULL");
+
+       media_format_get_audio_info(format, &mimetype, &channels, &rate, NULL, NULL);
+       media_format_unref(format);
+       RET_IF((type = __convert_media_format_mime_to_espp_mime(mimetype)) == -1, "not supported type");
+
+       conn->render.espp.audio_info.codec_data = NULL;
+       conn->render.espp.audio_info.codec_data_length = 0;
+       conn->render.espp.audio_info.bitrate = 0;
+       conn->render.espp.audio_info.channels = channels;
+       conn->render.espp.audio_info.sample_rate = rate;
+       conn->render.espp.audio_info.mime_type = type;
+}
+
+static void __espp_update_video_info(connection_s *conn, media_packet_h packet)
+{
+       media_format_h format;
+       media_format_mimetype_e mimetype;
+       int type;
+       int width = 0;
+       int height = 0;
+
+       RET_IF(!conn, "conn is NULL");
+       RET_IF(!packet, "packet is NULL");
+       RET_IF(media_packet_get_format(packet, &format) != MEDIA_PACKET_ERROR_NONE, "format is NULL");
+
+       media_format_get_video_info(format, &mimetype, &width, &height, NULL, NULL);
+       media_format_unref(format);
+       RET_IF((type = __convert_media_format_mime_to_espp_mime(mimetype)) == -1, "not supported type");
+
+       conn->render.espp.video_info.codec_data = NULL;
+       conn->render.espp.video_info.codec_data_length = 0;
+       conn->render.espp.video_info.width = width;
+       conn->render.espp.video_info.height = height;
+       conn->render.espp.video_info.max_width = width;
+       conn->render.espp.video_info.max_height = height;
+       conn->render.espp.video_info.mime_type = type;
+       conn->render.espp.video_info.framerate_num = 30; /* FIXME: avoid hard-coding */
+       conn->render.espp.video_info.framerate_den = 1;
+}
+#endif
+
 static void _webrtc_create(int index)
 {
        int ret = WEBRTC_ERROR_NONE;
@@ -441,16 +573,10 @@ static void _webrtc_destroy(int index)
        g_ad.conns[index].render.loopback_track_id = 0;
 
 #ifndef TIZEN_TV
-       if (g_ad.conns[index].encoded_audio_frame_cb_is_set)
-               g_ad.conns[index].encoded_audio_frame_cb_is_set = false;
+       g_ad.conns[index].encoded_audio_frame_cb_is_set = false;
+       g_ad.conns[index].encoded_video_frame_cb_is_set = false;
 
-       if (g_ad.conns[index].encoded_video_frame_cb_is_set)
-               g_ad.conns[index].encoded_video_frame_cb_is_set = false;
-
-       if (g_ad.conns[index].render.espp) {
-               g_ad.conns[index].render.espp = NULL;
-               g_print("espp close & destroy\n");
-       }
+       __espp_deinit(index);
 #endif
 }
 
@@ -466,16 +592,8 @@ static void _webrtc_start(int index)
        g_print("webrtc_start() success\n");
 #ifndef TIZEN_TV
        if (g_ad.conns[index].encoded_audio_frame_cb_is_set ||
-               g_ad.conns[index].encoded_video_frame_cb_is_set) {
-               if (!g_ad.conns[index].render.espp)
-                       g_print("espp create & open\n");
-       }
-
-       if (g_ad.conns[index].encoded_audio_frame_cb_is_set)
-               g_print("espp set audio stream info\n");
-
-       if (g_ad.conns[index].encoded_video_frame_cb_is_set)
-               g_print("espp set video stream info\n");
+               g_ad.conns[index].encoded_video_frame_cb_is_set)
+               __espp_init(index);
 #endif
 }
 
@@ -515,12 +633,8 @@ static void _webrtc_stop(int index)
 #endif
 #ifndef TIZEN_TV
        if (g_ad.conns[index].encoded_audio_frame_cb_is_set ||
-               g_ad.conns[index].encoded_video_frame_cb_is_set) {
-               if (g_ad.conns[index].render.espp) {
-                       g_ad.conns[index].render.espp = NULL;
-                       g_print("espp destroy\n");
-               }
-       }
+               g_ad.conns[index].encoded_video_frame_cb_is_set)
+               __espp_deinit(index);
 #endif
 }
 
@@ -2170,32 +2284,31 @@ static void __track_added_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned
 
        g_print("__track_added_cb() is invoked, webrtc[%p], type[%d], id[%u], conn[%p]\n", webrtc, type, id, conn);
 
-#ifndef TIZEN_TV
-       if (conn->encoded_audio_frame_cb_is_set && type == WEBRTC_MEDIA_TYPE_AUDIO) {
-               if (conn->render.espp) {
-                       /* espp: state check & prepare async */
-                       g_print("espp prepare async\n");
-               }
-               return;
-       }
-       if (conn->encoded_video_frame_cb_is_set && type == WEBRTC_MEDIA_TYPE_VIDEO) {
-               if (conn->render.espp) {
-                       /* espp: state check & prepare async */
-                       g_print("espp prepare async\n");
-               }
-               return;
-       }
-#endif
        if (type == WEBRTC_MEDIA_TYPE_VIDEO) {
 #ifndef __DEBUG_VALIDATE_ENCODED_FRAME_CB__
+#ifndef TIZEN_TV
+               conn->render.espp.video_track_preparing = true;
+#endif
                if (conn->render.display_type == WEBRTC_DISPLAY_TYPE_OVERLAY) {
                        g_print("Video track is added, set display - overlay, object[%p]\n", g_ad.win_id);
+#ifndef TIZEN_TV
+                       if (!conn->render.espp.handle)
+                               webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_OVERLAY, g_ad.win_id);
+                       else
+                               esplusplayer_set_display(conn->render.espp.handle, ESPLUSPLAYER_DISPLAY_TYPE_OVERLAY, g_ad.win_id);
+#else
                        webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_OVERLAY, g_ad.win_id);
-
+#endif
                } else if (conn->render.display_type == WEBRTC_DISPLAY_TYPE_EVAS) {
                        g_print("Video track is added, set display - evas object[%p]\n", conn->render.eo);
+#ifndef TIZEN_TV
+                       if (!conn->render.espp.handle)
+                               webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_EVAS, conn->render.eo);
+                       else
+                               esplusplayer_set_display(conn->render.espp.handle, ESPLUSPLAYER_DISPLAY_TYPE_EVAS, conn->render.eo);
+#else
                        webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_EVAS, conn->render.eo);
-
+#endif
                } else {
                        g_print("Video track is added, invalid display type[%d]\n", conn->render.display_type);
                }
@@ -2211,7 +2324,12 @@ static void __track_added_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned
                g_print("Audio track is added\n");
 #ifndef __DEBUG_VALIDATE_ENCODED_FRAME_CB__
                int ret;
+#ifndef TIZEN_TV
+               conn->render.espp.audio_track_preparing = true;
 
+               if (conn->render.espp.handle) /* TODO: apply stream info if the function is provided in espp */
+                       return;
+#endif
                if (!conn->render.stream_info) {
                        ret = sound_manager_create_stream_information(SOUND_STREAM_TYPE_MEDIA, NULL, NULL, &conn->render.stream_info);
                        RET_IF(ret != SOUND_MANAGER_ERROR_NONE, "failed to sound_manager_create_stream_information(), ret[0x%x]", ret);
@@ -2314,6 +2432,34 @@ static void _webrtc_unset_all_basic_cbs(int index)
                g_print("webrtc_unset_data_channel_cb() success\n");
 }
 
+#ifndef TIZEN_TV
+static void __espp_submit_packet(connection_s *conn, media_packet_h packet, webrtc_media_type_e type)
+{
+       uint64_t gst_pts;
+       uint64_t gst_duration;
+       esplusplayer_es_packet es_packet;
+       esplusplayer_submit_status submit_status;
+
+       RET_IF(!conn, "conn is NULL");
+       RET_IF(!packet, "media_packet is NULL");
+
+       es_packet.type = type == WEBRTC_MEDIA_TYPE_AUDIO ? ESPLUSPLAYER_STREAM_TYPE_AUDIO : ESPLUSPLAYER_STREAM_TYPE_VIDEO;
+       media_packet_get_buffer_data_ptr(packet, (void**)&es_packet.buffer);
+       media_packet_get_buffer_size(packet, (uint64_t *)&es_packet.buffer_size);
+       media_packet_get_pts(packet, &gst_pts);
+       media_packet_get_duration(packet, &gst_duration);
+       if (GST_CLOCK_TIME_IS_VALID(gst_pts))
+               es_packet.pts = GST_TIME_AS_MSECONDS(gst_pts);
+       if (GST_CLOCK_TIME_IS_VALID(gst_duration))
+               es_packet.duration = GST_TIME_AS_MSECONDS(gst_duration);
+       es_packet.matroska_color_info = NULL;
+       es_packet.hdr10p_metadata = NULL;
+
+       submit_status = esplusplayer_submit_packet(conn->render.espp.handle, &es_packet);
+       g_print("packet[media:%p, es:%p] type[%d] espp submit packet status[%d]\n", packet, &es_packet, type, submit_status);
+}
+#endif
+
 static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned int track_id, media_packet_h packet, void *user_data)
 {
        void *data_ptr = NULL;
@@ -2329,20 +2475,83 @@ static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsign
                webrtc, type, track_id, packet, data_ptr, user_data);
 #ifndef TIZEN_TV
        if (type == WEBRTC_MEDIA_TYPE_AUDIO && conn->encoded_audio_frame_cb_is_set) {
-               if (!conn->render.espp) {
+               if (!conn->render.espp.handle) {
                        g_printerr("render.espp is NULL\n");
-                       media_packet_destroy(packet);
-                       return;
+                       goto out;
+               }
+               /* Get format and set espp audiostream & prepare async */
+               if (conn->render.espp.audio_track_preparing) {
+                       __espp_update_audio_info(conn, packet);
+
+                       esplusplayer_set_audio_stream_info(conn->render.espp.handle, &conn->render.espp.audio_info);
+                       esplusplayer_activate(conn->render.espp.handle, ESPLUSPLAYER_STREAM_TYPE_AUDIO);
+
+                       if (!conn->render.espp.video_track_preparing) {
+                               g_print("espp prepare async\n");
+                               esplusplayer_prepare_async(conn->render.espp.handle);
+
+                               g_mutex_lock(&conn->render.espp.mutex);
+                               g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for ready to prepare cb */
+                               g_mutex_unlock(&conn->render.espp.mutex);
+
+                               __espp_submit_packet(conn, packet, type);
+
+                               g_mutex_lock(&conn->render.espp.mutex);
+                               g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for prepare async done cb */
+                               g_mutex_unlock(&conn->render.espp.mutex);
+
+                               g_print("espp start\n");
+                               esplusplayer_start(conn->render.espp.handle);
+                       }
+
+                       conn->render.espp.audio_track_preparing = false;
+                       goto out;
                }
-               /* TODO: submit data to espp */
+
+               if (conn->render.espp.video_track_preparing) /* not ready for video track, skip submit packet */
+                       goto out;
+
+               __espp_submit_packet(conn, packet, type);
 
        } else if (type == WEBRTC_MEDIA_TYPE_VIDEO && conn->encoded_video_frame_cb_is_set) {
-               if (!conn->render.espp) {
+
+               if (!conn->render.espp.handle) {
                        g_printerr("render.espp is NULL\n");
-                       media_packet_destroy(packet);
-                       return;
+                       goto out;
+               }
+               /* Get format and set espp videostream & prepare async */
+               if (conn->render.espp.video_track_preparing) {
+                       __espp_update_video_info(conn, packet);
+
+                       esplusplayer_set_video_stream_info(conn->render.espp.handle, &conn->render.espp.video_info);
+                       esplusplayer_activate(conn->render.espp.handle, ESPLUSPLAYER_STREAM_TYPE_VIDEO);
+
+                       if (!conn->render.espp.audio_track_preparing) {
+                               g_print("espp prepare async\n");
+                               esplusplayer_prepare_async(conn->render.espp.handle);
+
+                               g_mutex_lock(&conn->render.espp.mutex);
+                               g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for ready to prepare cb */
+                               g_mutex_unlock(&conn->render.espp.mutex);
+
+                               __espp_submit_packet(conn, packet, type);
+
+                               g_mutex_lock(&conn->render.espp.mutex);
+                               g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for prepare async done cb */
+                               g_mutex_unlock(&conn->render.espp.mutex);
+
+                               g_print("espp start\n");
+                               esplusplayer_start(conn->render.espp.handle);
+                       }
+
+                       conn->render.espp.video_track_preparing = false;
+                       goto out;
                }
-               /* TODO: submit data to espp */
+
+               if (conn->render.espp.audio_track_preparing) /* not ready for audio track, skip submit packet */
+                       goto out;
+
+               __espp_submit_packet(conn, packet, type);
        }
 #endif
 #ifdef __DEBUG_VALIDATE_ENCODED_FRAME_CB__
@@ -2408,6 +2617,9 @@ static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsign
                gst_buffer_unref(buffer);
        }
 #endif /* __DEBUG_VALIDATE_ENCODED_FRAME_CB__ */
+#ifndef TIZEN_TV
+out:
+#endif
        /* media packet should be freed after use */
        media_packet_destroy(packet);
 }
@@ -2424,10 +2636,7 @@ static void _webrtc_set_encoded_audio_frame_cb(int index)
 #ifndef TIZEN_TV
        if (!g_ad.conns[index].encoded_audio_frame_cb_is_set) {
                g_ad.conns[index].encoded_audio_frame_cb_is_set = true;
-               if (!g_ad.conns[index].render.espp)
-                       g_print("espp create & open\n");
-
-               g_print("espp set audio stream info\n");
+               __espp_init(index);
        }
 #endif
 #endif
@@ -2445,7 +2654,7 @@ static void _webrtc_unset_encoded_audio_frame_cb(int index)
 #ifndef TIZEN_TV
        if (g_ad.conns[index].encoded_audio_frame_cb_is_set) {
                g_ad.conns[index].encoded_audio_frame_cb_is_set = false;
-               g_print("espp close & destroy\n");
+               __espp_deinit(index);
        }
 #endif
 #endif
@@ -2464,10 +2673,7 @@ static void _webrtc_set_encoded_video_frame_cb(int index)
 #ifndef TIZEN_TV
        if (!g_ad.conns[index].encoded_video_frame_cb_is_set) {
                g_ad.conns[index].encoded_video_frame_cb_is_set = true;
-               if (!g_ad.conns[index].render.espp) {
-                       g_print("espp create & open\n");
-               }
-               g_print("espp set video stream info\n");
+               __espp_init(index);
        }
 #endif
 #endif
@@ -2486,7 +2692,7 @@ static void _webrtc_unset_encoded_video_frame_cb(int index)
 #ifndef TIZEN_TV
        if (g_ad.conns[index].encoded_video_frame_cb_is_set) {
                g_ad.conns[index].encoded_video_frame_cb_is_set = false;
-               g_print("espp close & destroy\n");
+               __espp_deinit(index);
        }
 #endif
 #endif

diff --git a/test/webrtc_test_priv.h b/test/webrtc_test_priv.h
index d824b1f73492f0be563d66375c06dc40ae406b5d..44bfd56a3785c392836c78a975f0e1fc9d57e891 100644 (file)
--- a/test/webrtc_test_priv.h
+++ b/test/webrtc_test_priv.h
@@ -222,7 +222,15 @@ typedef struct _connection_s {
                Evas_Object *text_eo;
                unsigned int loopback_track_id;
 #ifndef TIZEN_TV
-               esplusplayer_handle espp;
+               struct {
+                       esplusplayer_handle handle;
+                       esplusplayer_audio_stream_info audio_info;
+                       esplusplayer_video_stream_info video_info;
+                       bool audio_track_preparing;
+                       bool video_track_preparing;
+                       GCond cond;
+                       GMutex mutex;
+               } espp;
 #endif
        } render;