From: Sangchul Lee Date: Fri, 1 Oct 2021 08:52:35 +0000 (+0900) Subject: webrtc_test: esplusplayer integration X-Git-Tag: submit/tizen/20220706.020113~9 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=9e41501ffbc988cc857c0162d52a37161c4b7876;p=platform%2Fcore%2Fapi%2Fwebrtc.git webrtc_test: esplusplayer integration The esplusplayer will be activated to render received data if an encoded frame callback is set. Use commands below. 'sa'. Set encoded audio frame callback 'sv'. Set encoded video frame callback [Version] 0.3.136 [Issue Type] New feature Change-Id: I1400be4a77b6b99db44788dcf428a0e969e9571f Signed-off-by: Sangchul Lee --- diff --git a/packaging/capi-media-webrtc.spec b/packaging/capi-media-webrtc.spec index 21315a2f..f61a93e2 100644 --- a/packaging/capi-media-webrtc.spec +++ b/packaging/capi-media-webrtc.spec @@ -1,6 +1,6 @@ Name: capi-media-webrtc Summary: A WebRTC library in Tizen Native API -Version: 0.3.135 +Version: 0.3.136 Release: 0 Group: Multimedia/API License: Apache-2.0 diff --git a/test/webrtc_test.c b/test/webrtc_test.c index 8f5b6174..5b8afc4a 100644 --- a/test/webrtc_test.c +++ b/test/webrtc_test.c @@ -374,6 +374,138 @@ static void __release_packet_source(int conn_idx, unsigned int source_idx) g_cond_clear(&packet_source->cond); } +#ifndef TIZEN_TV +static void __espp_prepare_async_done_cb(bool result, void *user_data) +{ + connection_s *conn = (connection_s *)user_data; + + g_print("__espp_prepare_async_done_cb() is called, result[%u] conn[%p]\n", result, conn); + + g_cond_signal(&conn->render.espp.cond); +} + +static void __espp_ready_to_prepare_cb(esplusplayer_stream_type type, void *user_data) +{ + connection_s *conn = (connection_s *)user_data; + + g_print("__espp_ready_to_prepare_cb() is called, type[%u], conn[%p]\n", type, conn); + + g_cond_signal(&conn->render.espp.cond); +} + +static void __espp_init(int index) +{ + if (g_ad.conns[index].render.espp.handle) + return; + + g_mutex_init(&g_ad.conns[index].render.espp.mutex); + g_cond_init(&g_ad.conns[index].render.espp.cond); + g_ad.conns[index].render.espp.audio_track_preparing = false; + g_ad.conns[index].render.espp.video_track_preparing = false; + + g_print("espp create & open\n"); + g_ad.conns[index].render.espp.handle = esplusplayer_create(); + esplusplayer_set_prepare_async_done_cb(g_ad.conns[index].render.espp.handle, __espp_prepare_async_done_cb, &g_ad.conns[index]); + esplusplayer_set_ready_to_prepare_cb(g_ad.conns[index].render.espp.handle, __espp_ready_to_prepare_cb, &g_ad.conns[index]); + esplusplayer_open(g_ad.conns[index].render.espp.handle); +} + +static void __espp_deinit(int index) +{ + if (!g_ad.conns[index].render.espp.handle) + return; + + g_print("espp stop & close & destroy\n"); + esplusplayer_stop(g_ad.conns[index].render.espp.handle); + esplusplayer_close(g_ad.conns[index].render.espp.handle); + esplusplayer_destroy(g_ad.conns[index].render.espp.handle); + + g_ad.conns[index].render.espp.handle = NULL; + g_ad.conns[index].render.espp.audio_track_preparing = false; + g_ad.conns[index].render.espp.video_track_preparing = false; + g_mutex_clear(&g_ad.conns[index].render.espp.mutex); + g_cond_clear(&g_ad.conns[index].render.espp.cond); +} + +static int __convert_media_format_mime_to_espp_mime(media_format_mimetype_e type) +{ + g_print("media_format_mimetype(0x%x)\n", type); + + switch (type) { + case MEDIA_FORMAT_OPUS: + return (int)ESPLUSPLAYER_AUDIO_MIME_TYPE_OPUS; + + case MEDIA_FORMAT_VP8: + return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_VP8; + + case MEDIA_FORMAT_VP9: + return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_VP9; + + case MEDIA_FORMAT_H264_SP: + case MEDIA_FORMAT_H264_MP: + case MEDIA_FORMAT_H264_HP: + return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_H264; + + case MEDIA_FORMAT_MJPEG: + return (int)ESPLUSPLAYER_VIDEO_MIME_TYPE_MJPEG; + + default: + return -1; + } +} + +static void __espp_update_audio_info(connection_s *conn, media_packet_h packet) +{ + media_format_h format; + media_format_mimetype_e mimetype; + int type; + int channels = 0; + int rate = 0; + + RET_IF(!conn, "conn is NULL"); + RET_IF(!packet, "packet is NULL"); + RET_IF(media_packet_get_format(packet, &format) != MEDIA_PACKET_ERROR_NONE, "format is NULL"); + + media_format_get_audio_info(format, &mimetype, &channels, &rate, NULL, NULL); + media_format_unref(format); + RET_IF((type = __convert_media_format_mime_to_espp_mime(mimetype)) == -1, "not supported type"); + + conn->render.espp.audio_info.codec_data = NULL; + conn->render.espp.audio_info.codec_data_length = 0; + conn->render.espp.audio_info.bitrate = 0; + conn->render.espp.audio_info.channels = channels; + conn->render.espp.audio_info.sample_rate = rate; + conn->render.espp.audio_info.mime_type = type; +} + +static void __espp_update_video_info(connection_s *conn, media_packet_h packet) +{ + media_format_h format; + media_format_mimetype_e mimetype; + int type; + int width = 0; + int height = 0; + + RET_IF(!conn, "conn is NULL"); + RET_IF(!packet, "packet is NULL"); + RET_IF(media_packet_get_format(packet, &format) != MEDIA_PACKET_ERROR_NONE, "format is NULL"); + + media_format_get_video_info(format, &mimetype, &width, &height, NULL, NULL); + media_format_unref(format); + RET_IF((type = __convert_media_format_mime_to_espp_mime(mimetype)) == -1, "not supported type"); + + conn->render.espp.video_info.codec_data = NULL; + conn->render.espp.video_info.codec_data_length = 0; + conn->render.espp.video_info.width = width; + conn->render.espp.video_info.height = height; + conn->render.espp.video_info.max_width = width; + conn->render.espp.video_info.max_height = height; + conn->render.espp.video_info.mime_type = type; + conn->render.espp.video_info.framerate_num = 30; /* FIXME: avoid hard-coding */ + conn->render.espp.video_info.framerate_den = 1; +} +#endif + static void _webrtc_create(int index) { int ret = WEBRTC_ERROR_NONE; @@ -441,16 +573,10 @@ static void _webrtc_destroy(int index) g_ad.conns[index].render.loopback_track_id = 0; #ifndef TIZEN_TV - if (g_ad.conns[index].encoded_audio_frame_cb_is_set) - g_ad.conns[index].encoded_audio_frame_cb_is_set = false; + g_ad.conns[index].encoded_audio_frame_cb_is_set = false; + g_ad.conns[index].encoded_video_frame_cb_is_set = false; - if (g_ad.conns[index].encoded_video_frame_cb_is_set) - g_ad.conns[index].encoded_video_frame_cb_is_set = false; - - if (g_ad.conns[index].render.espp) { - g_ad.conns[index].render.espp = NULL; - g_print("espp close & destroy\n"); - } + __espp_deinit(index); #endif } @@ -466,16 +592,8 @@ static void _webrtc_start(int index) g_print("webrtc_start() success\n"); #ifndef TIZEN_TV if (g_ad.conns[index].encoded_audio_frame_cb_is_set || - g_ad.conns[index].encoded_video_frame_cb_is_set) { - if (!g_ad.conns[index].render.espp) - g_print("espp create & open\n"); - } - - if (g_ad.conns[index].encoded_audio_frame_cb_is_set) - g_print("espp set audio stream info\n"); - - if (g_ad.conns[index].encoded_video_frame_cb_is_set) - g_print("espp set video stream info\n"); + g_ad.conns[index].encoded_video_frame_cb_is_set) + __espp_init(index); #endif } @@ -515,12 +633,8 @@ static void _webrtc_stop(int index) #endif #ifndef TIZEN_TV if (g_ad.conns[index].encoded_audio_frame_cb_is_set || - g_ad.conns[index].encoded_video_frame_cb_is_set) { - if (g_ad.conns[index].render.espp) { - g_ad.conns[index].render.espp = NULL; - g_print("espp destroy\n"); - } - } + g_ad.conns[index].encoded_video_frame_cb_is_set) + __espp_deinit(index); #endif } @@ -2170,32 +2284,31 @@ static void __track_added_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned g_print("__track_added_cb() is invoked, webrtc[%p], type[%d], id[%u], conn[%p]\n", webrtc, type, id, conn); -#ifndef TIZEN_TV - if (conn->encoded_audio_frame_cb_is_set && type == WEBRTC_MEDIA_TYPE_AUDIO) { - if (conn->render.espp) { - /* espp: state check & prepare async */ - g_print("espp prepare async\n"); - } - return; - } - if (conn->encoded_video_frame_cb_is_set && type == WEBRTC_MEDIA_TYPE_VIDEO) { - if (conn->render.espp) { - /* espp: state check & prepare async */ - g_print("espp prepare async\n"); - } - return; - } -#endif if (type == WEBRTC_MEDIA_TYPE_VIDEO) { #ifndef __DEBUG_VALIDATE_ENCODED_FRAME_CB__ +#ifndef TIZEN_TV + conn->render.espp.video_track_preparing = true; +#endif if (conn->render.display_type == WEBRTC_DISPLAY_TYPE_OVERLAY) { g_print("Video track is added, set display - overlay, object[%p]\n", g_ad.win_id); +#ifndef TIZEN_TV + if (!conn->render.espp.handle) + webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_OVERLAY, g_ad.win_id); + else + esplusplayer_set_display(conn->render.espp.handle, ESPLUSPLAYER_DISPLAY_TYPE_OVERLAY, g_ad.win_id); +#else webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_OVERLAY, g_ad.win_id); - +#endif } else if (conn->render.display_type == WEBRTC_DISPLAY_TYPE_EVAS) { g_print("Video track is added, set display - evas object[%p]\n", conn->render.eo); +#ifndef TIZEN_TV + if (!conn->render.espp.handle) + webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_EVAS, conn->render.eo); + else + esplusplayer_set_display(conn->render.espp.handle, ESPLUSPLAYER_DISPLAY_TYPE_EVAS, conn->render.eo); +#else webrtc_set_display(webrtc, id, WEBRTC_DISPLAY_TYPE_EVAS, conn->render.eo); - +#endif } else { g_print("Video track is added, invalid display type[%d]\n", conn->render.display_type); } @@ -2211,7 +2324,12 @@ static void __track_added_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned g_print("Audio track is added\n"); #ifndef __DEBUG_VALIDATE_ENCODED_FRAME_CB__ int ret; +#ifndef TIZEN_TV + conn->render.espp.audio_track_preparing = true; + if (conn->render.espp.handle) /* TODO: apply stream info if the function is provided in espp */ + return; +#endif if (!conn->render.stream_info) { ret = sound_manager_create_stream_information(SOUND_STREAM_TYPE_MEDIA, NULL, NULL, &conn->render.stream_info); RET_IF(ret != SOUND_MANAGER_ERROR_NONE, "failed to sound_manager_create_stream_information(), ret[0x%x]", ret); @@ -2314,6 +2432,34 @@ static void _webrtc_unset_all_basic_cbs(int index) g_print("webrtc_unset_data_channel_cb() success\n"); } +#ifndef TIZEN_TV +static void __espp_submit_packet(connection_s *conn, media_packet_h packet, webrtc_media_type_e type) +{ + uint64_t gst_pts; + uint64_t gst_duration; + esplusplayer_es_packet es_packet; + esplusplayer_submit_status submit_status; + + RET_IF(!conn, "conn is NULL"); + RET_IF(!packet, "media_packet is NULL"); + + es_packet.type = type == WEBRTC_MEDIA_TYPE_AUDIO ? ESPLUSPLAYER_STREAM_TYPE_AUDIO : ESPLUSPLAYER_STREAM_TYPE_VIDEO; + media_packet_get_buffer_data_ptr(packet, (void**)&es_packet.buffer); + media_packet_get_buffer_size(packet, (uint64_t *)&es_packet.buffer_size); + media_packet_get_pts(packet, &gst_pts); + media_packet_get_duration(packet, &gst_duration); + if (GST_CLOCK_TIME_IS_VALID(gst_pts)) + es_packet.pts = GST_TIME_AS_MSECONDS(gst_pts); + if (GST_CLOCK_TIME_IS_VALID(gst_duration)) + es_packet.duration = GST_TIME_AS_MSECONDS(gst_duration); + es_packet.matroska_color_info = NULL; + es_packet.hdr10p_metadata = NULL; + + submit_status = esplusplayer_submit_packet(conn->render.espp.handle, &es_packet); + g_print("packet[media:%p, es:%p] type[%d] espp submit packet status[%d]\n", packet, &es_packet, type, submit_status); +} +#endif + static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsigned int track_id, media_packet_h packet, void *user_data) { void *data_ptr = NULL; @@ -2329,20 +2475,83 @@ static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsign webrtc, type, track_id, packet, data_ptr, user_data); #ifndef TIZEN_TV if (type == WEBRTC_MEDIA_TYPE_AUDIO && conn->encoded_audio_frame_cb_is_set) { - if (!conn->render.espp) { + if (!conn->render.espp.handle) { g_printerr("render.espp is NULL\n"); - media_packet_destroy(packet); - return; + goto out; + } + /* Get format and set espp audiostream & prepare async */ + if (conn->render.espp.audio_track_preparing) { + __espp_update_audio_info(conn, packet); + + esplusplayer_set_audio_stream_info(conn->render.espp.handle, &conn->render.espp.audio_info); + esplusplayer_activate(conn->render.espp.handle, ESPLUSPLAYER_STREAM_TYPE_AUDIO); + + if (!conn->render.espp.video_track_preparing) { + g_print("espp prepare async\n"); + esplusplayer_prepare_async(conn->render.espp.handle); + + g_mutex_lock(&conn->render.espp.mutex); + g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for ready to prepare cb */ + g_mutex_unlock(&conn->render.espp.mutex); + + __espp_submit_packet(conn, packet, type); + + g_mutex_lock(&conn->render.espp.mutex); + g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for prepare async done cb */ + g_mutex_unlock(&conn->render.espp.mutex); + + g_print("espp start\n"); + esplusplayer_start(conn->render.espp.handle); + } + + conn->render.espp.audio_track_preparing = false; + goto out; } - /* TODO: submit data to espp */ + + if (conn->render.espp.video_track_preparing) /* not ready for video track, skip submit packet */ + goto out; + + __espp_submit_packet(conn, packet, type); } else if (type == WEBRTC_MEDIA_TYPE_VIDEO && conn->encoded_video_frame_cb_is_set) { - if (!conn->render.espp) { + + if (!conn->render.espp.handle) { g_printerr("render.espp is NULL\n"); - media_packet_destroy(packet); - return; + goto out; + } + /* Get format and set espp videostream & prepare async */ + if (conn->render.espp.video_track_preparing) { + __espp_update_video_info(conn, packet); + + esplusplayer_set_video_stream_info(conn->render.espp.handle, &conn->render.espp.video_info); + esplusplayer_activate(conn->render.espp.handle, ESPLUSPLAYER_STREAM_TYPE_VIDEO); + + if (!conn->render.espp.audio_track_preparing) { + g_print("espp prepare async\n"); + esplusplayer_prepare_async(conn->render.espp.handle); + + g_mutex_lock(&conn->render.espp.mutex); + g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for ready to prepare cb */ + g_mutex_unlock(&conn->render.espp.mutex); + + __espp_submit_packet(conn, packet, type); + + g_mutex_lock(&conn->render.espp.mutex); + g_cond_wait(&conn->render.espp.cond, &conn->render.espp.mutex); /* wait for prepare async done cb */ + g_mutex_unlock(&conn->render.espp.mutex); + + g_print("espp start\n"); + esplusplayer_start(conn->render.espp.handle); + } + + conn->render.espp.video_track_preparing = false; + goto out; } - /* TODO: submit data to espp */ + + if (conn->render.espp.audio_track_preparing) /* not ready for audio track, skip submit packet */ + goto out; + + __espp_submit_packet(conn, packet, type); } #endif #ifdef __DEBUG_VALIDATE_ENCODED_FRAME_CB__ @@ -2408,6 +2617,9 @@ static void __encoded_frame_cb(webrtc_h webrtc, webrtc_media_type_e type, unsign gst_buffer_unref(buffer); } #endif /* __DEBUG_VALIDATE_ENCODED_FRAME_CB__ */ +#ifndef TIZEN_TV +out: +#endif /* media packet should be freed after use */ media_packet_destroy(packet); } @@ -2424,10 +2636,7 @@ static void _webrtc_set_encoded_audio_frame_cb(int index) #ifndef TIZEN_TV if (!g_ad.conns[index].encoded_audio_frame_cb_is_set) { g_ad.conns[index].encoded_audio_frame_cb_is_set = true; - if (!g_ad.conns[index].render.espp) - g_print("espp create & open\n"); - - g_print("espp set audio stream info\n"); + __espp_init(index); } #endif #endif @@ -2445,7 +2654,7 @@ static void _webrtc_unset_encoded_audio_frame_cb(int index) #ifndef TIZEN_TV if (g_ad.conns[index].encoded_audio_frame_cb_is_set) { g_ad.conns[index].encoded_audio_frame_cb_is_set = false; - g_print("espp close & destroy\n"); + __espp_deinit(index); } #endif #endif @@ -2464,10 +2673,7 @@ static void _webrtc_set_encoded_video_frame_cb(int index) #ifndef TIZEN_TV if (!g_ad.conns[index].encoded_video_frame_cb_is_set) { g_ad.conns[index].encoded_video_frame_cb_is_set = true; - if (!g_ad.conns[index].render.espp) { - g_print("espp create & open\n"); - } - g_print("espp set video stream info\n"); + __espp_init(index); } #endif #endif @@ -2486,7 +2692,7 @@ static void _webrtc_unset_encoded_video_frame_cb(int index) #ifndef TIZEN_TV if (g_ad.conns[index].encoded_video_frame_cb_is_set) { g_ad.conns[index].encoded_video_frame_cb_is_set = false; - g_print("espp close & destroy\n"); + __espp_deinit(index); } #endif #endif diff --git a/test/webrtc_test_priv.h b/test/webrtc_test_priv.h index d824b1f7..44bfd56a 100644 --- a/test/webrtc_test_priv.h +++ b/test/webrtc_test_priv.h @@ -222,7 +222,15 @@ typedef struct _connection_s { Evas_Object *text_eo; unsigned int loopback_track_id; #ifndef TIZEN_TV - esplusplayer_handle espp; + struct { + esplusplayer_handle handle; + esplusplayer_audio_stream_info audio_info; + esplusplayer_video_stream_info video_info; + bool audio_track_preparing; + bool video_track_preparing; + GCond cond; + GMutex mutex; + } espp; #endif } render;