From 6f0e71491ef0ddf824498323cd207055db7b2f5b Mon Sep 17 00:00:00 2001 From: Eunhye Choi Date: Wed, 17 Apr 2019 21:32:47 +0900 Subject: [PATCH] [0.6.183] support pcm extraction with audio playback - add new audio extract option to support playback (experimental) - reconfigure the audio bin structure Change-Id: Ieb21849b62e2d83196af5028533594db1f22d39a --- src/include/mm_player.h | 10 +- src/include/mm_player_priv.h | 10 +- src/include/mm_player_utils.h | 15 ++ src/mm_player_capture.c | 1 + src/mm_player_ini.c | 2 +- src/mm_player_priv.c | 496 ++++++++++++++++++++++++++---------------- src/mm_player_utils.c | 2 + 7 files changed, 339 insertions(+), 197 deletions(-) diff --git a/src/include/mm_player.h b/src/include/mm_player.h index 3d0e34e..e4fbacd 100644 --- a/src/include/mm_player.h +++ b/src/include/mm_player.h @@ -383,10 +383,12 @@ typedef enum { } mmplayer_video_codec_type_e; typedef enum { - MM_PLAYER_AUDIO_EXTRACT_DEFAULT = 0x00, /**< Sync with the playback clock and multichannel audio stream */ - MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK = 0x01, /**< No sync with the playback clock */ - MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE = 0x02, /**< Splits one interleaved multichannel audio stream into many mono audio streams */ - MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_AND_DEINTERLEAVE = MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK | MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE, + MM_PLAYER_AUDIO_EXTRACT_DEFAULT = 0x00, /**< Sync with the playback clock and multichannel audio stream */ + MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK = 0x01, /**< No sync with the playback clock */ + MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE = 0x02, /**< Splits one interleaved multichannel audio stream into many mono audio streams */ + MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_AND_DEINTERLEAVE = MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK | MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE, + MM_PLAYER_AUDIO_EXTRACT_WITH_PLAYBACK = 0x04, /**< With audio playback synchronously (experimental) */ + MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE_WITH_PLAYBACK = MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE | MM_PLAYER_AUDIO_EXTRACT_WITH_PLAYBACK, } mmplayer_audio_extract_opt_e; /** diff --git a/src/include/mm_player_priv.h b/src/include/mm_player_priv.h index 2182cb3..d1d4e37 100644 --- a/src/include/mm_player_priv.h +++ b/src/include/mm_player_priv.h @@ -201,15 +201,21 @@ enum audio_element_id { MMPLAYER_A_VOL, MMPLAYER_A_FILTER, MMPLAYER_A_FILTER_SEC, - MMPLAYER_A_CAPS_DEFAULT, MMPLAYER_A_CONV_BFORMAT, MMPLAYER_A_CAPS_360, MMPLAYER_A_SINK, MMPLAYER_A_RESAMPLER, - MMPLAYER_A_DEINTERLEAVE, MMPLAYER_A_RGVOL, MMPLAYER_A_CONV_PITCH, MMPLAYER_A_PITCH, + MMPLAYER_A_TEE, + MMPLAYER_A_TEE_Q1, + MMPLAYER_A_TEE_Q2, + MMPLAYER_A_EXTRACT_CONV, + MMPLAYER_A_EXTRACT_RESAMPLER, + MMPLAYER_A_EXTRACT_CAPS, + MMPLAYER_A_EXTRACT_DEINTERLEAVE, + MMPLAYER_A_EXTRACT_SINK, MMPLAYER_A_NUM }; diff --git a/src/include/mm_player_utils.h b/src/include/mm_player_utils.h index be16503..34833cd 100644 --- a/src/include/mm_player_utils.h +++ b/src/include/mm_player_utils.h @@ -227,6 +227,21 @@ } \ } while (0) +/* create element */ +#define MMPLAYER_CREATE_ELEMENT(x_bin, x_id, x_factory, x_name, x_bucket, x_player) \ + do {\ + x_bin[x_id].id = x_id;\ + x_bin[x_id].gst = gst_element_factory_make(x_factory, x_name);\ + if (!x_bin[x_id].gst) {\ + LOGE("failed to create %s", x_factory);\ + goto ERROR;\ + } else {\ + if (x_player->ini.set_dump_element_flag)\ + __mmplayer_add_dump_buffer_probe(x_player, x_bin[x_id].gst);\ + } \ + x_bucket = g_list_append(x_bucket, &x_bin[x_id]);\ + } while (0); + /* release element resource */ #define MMPLAYER_RELEASE_ELEMENT(x_player, x_bin, x_id) \ do { \ diff --git a/src/mm_player_capture.c b/src/mm_player_capture.c index 8b71aaa..0e11f7d 100644 --- a/src/mm_player_capture.c +++ b/src/mm_player_capture.c @@ -426,6 +426,7 @@ __mmplayer_get_video_frame_from_buffer(mmplayer_t *player, GstPad *pad, GstBuffe plane_size = player->captured.stride_width[i] * player->captured.stride_height[i]; if (!plane_size) { LOGE("invalid plane size"); + gst_video_frame_unmap(&vframe); return MM_ERROR_PLAYER_INTERNAL; } pixels = GST_VIDEO_FRAME_PLANE_DATA(&vframe, i); diff --git a/src/mm_player_ini.c b/src/mm_player_ini.c index 5bdec47..34a9dd6 100644 --- a/src/mm_player_ini.c +++ b/src/mm_player_ini.c @@ -291,7 +291,7 @@ mm_player_ini_load(mmplayer_ini_t *ini) LOGD("dump_element_keyword [%d] : %s", idx, ini->dump_element_keyword[idx]); for (idx = 0; ini->unsupported_codec_keyword[idx][0] != '\0'; idx++) - LOGD("unsupported_codec_keyword [%d] : %s", idx, ini->dump_element_keyword[idx]); + LOGD("unsupported_codec_keyword [%d] : %s", idx, ini->unsupported_codec_keyword[idx]); /* http streaming */ LOGD("httpsrc element : %s", ini->httpsrc_element); diff --git a/src/mm_player_priv.c b/src/mm_player_priv.c index 4f65f40..3e4f41e 100644 --- a/src/mm_player_priv.c +++ b/src/mm_player_priv.c @@ -705,17 +705,11 @@ __mmplayer_gst_remove_fakesink(mmplayer_t *player, mmplayer_gst_element_t *fakes GstElement *parent = NULL; MMPLAYER_RETURN_VAL_IF_FAIL(player && player->pipeline, FALSE); - - /* if we have no fakesink. this meas we are using decodebin which doesn' - t need to add extra fakesink */ - MMPLAYER_RETURN_VAL_IF_FAIL(fakesink, TRUE); + MMPLAYER_RETURN_VAL_IF_FAIL(fakesink && fakesink->gst, TRUE); /* lock */ MMPLAYER_FSINK_LOCK(player); - if (!fakesink->gst) - goto ERROR; - /* get parent of fakesink */ parent = (GstElement *)gst_object_get_parent((GstObject *)fakesink->gst); if (!parent) { @@ -2113,6 +2107,7 @@ __mmplayer_gst_element_link_bucket(GList *element_bucket) GList *bucket = element_bucket; mmplayer_gst_element_t *element = NULL; mmplayer_gst_element_t *prv_element = NULL; + GstElement *tee_element = NULL; gint successful_link_count = 0; MMPLAYER_FENTER(); @@ -2127,11 +2122,25 @@ __mmplayer_gst_element_link_bucket(GList *element_bucket) if (element && element->gst) { if (prv_element && prv_element->gst) { + if (strstr(GST_ELEMENT_NAME(element->gst), "audio-tee-queue") && strcmp(GST_ELEMENT_NAME(prv_element->gst), "audio-tee")) { + if (tee_element) { + prv_element->gst = tee_element; + } else { + LOGD("failed to make new audio branch - linking [%s] to [%s] is not supported", + GST_ELEMENT_NAME(GST_ELEMENT(prv_element->gst)), + GST_ELEMENT_NAME(GST_ELEMENT(element->gst))); + return -1; + } + } if (gst_element_link(GST_ELEMENT(prv_element->gst), GST_ELEMENT(element->gst))) { LOGD("linking [%s] to [%s] success", GST_ELEMENT_NAME(GST_ELEMENT(prv_element->gst)), GST_ELEMENT_NAME(GST_ELEMENT(element->gst))); successful_link_count++; + if (!strcmp(GST_ELEMENT_NAME(prv_element->gst), "audio-tee")) { + LOGD("keep audio-tee element for next audio pipeline branch"); + tee_element = prv_element->gst; + } } else { LOGD("linking [%s] to [%s] failed", GST_ELEMENT_NAME(GST_ELEMENT(prv_element->gst)), @@ -2235,31 +2244,6 @@ ERROR: return; } -/** - * This function is to create audio pipeline for playing. - * - * @param player [in] handle of player - * - * @return This function returns zero on success. - * @remark - * @see __mmplayer_gst_create_midi_pipeline, __mmplayer_gst_create_video_sink_bin - */ -/* macro for code readability. just for sinkbin-creation functions */ -#define MMPLAYER_CREATE_ELEMENT(x_bin, x_id, x_factory, x_name, x_add_bucket, x_player) \ - do {\ - x_bin[x_id].id = x_id;\ - x_bin[x_id].gst = gst_element_factory_make(x_factory, x_name);\ - if (!x_bin[x_id].gst) {\ - LOGE("failed to create %s", x_factory);\ - goto ERROR;\ - } else {\ - if (x_player->ini.set_dump_element_flag)\ - __mmplayer_add_dump_buffer_probe(x_player, x_bin[x_id].gst);\ - } \ - if (x_add_bucket)\ - element_bucket = g_list_append(element_bucket, &x_bin[x_id]);\ - } while (0); - void __mmplayer_audio_stream_clear_buffer(mmplayer_t *player, gboolean send_all) { @@ -2463,6 +2447,12 @@ __mmplayer_gst_audio_deinterleave_pad_added(GstElement *elem, GstPad *pad, gpoin g_object_set(sink, "sync", TRUE, NULL); g_object_set(sink, "signal-handoffs", TRUE, NULL); + /* keep the first sink reference only */ + if (!audiobin[MMPLAYER_A_SINK].gst) { + audiobin[MMPLAYER_A_SINK].id = MMPLAYER_A_SINK; + audiobin[MMPLAYER_A_SINK].gst = sink; + } + gst_element_set_state(sink, GST_STATE_PAUSED); gst_element_set_state(queue, GST_STATE_PAUSED); @@ -2473,6 +2463,8 @@ __mmplayer_gst_audio_deinterleave_pad_added(GstElement *elem, GstPad *pad, gpoin G_CALLBACK(__mmplayer_audio_stream_decoded_render_cb), (gpointer)player); + __mmplayer_add_sink(player, sink); + MMPLAYER_FLEAVE(); return; @@ -2495,7 +2487,7 @@ ERROR: } void -__mmplayer_gst_set_pulsesink_property(mmplayer_t *player, MMHandleType attrs) +__mmplayer_gst_set_pulsesink_property(mmplayer_t *player) { #define MAX_PROPS_LEN 128 gint latency_mode = 0; @@ -2512,8 +2504,8 @@ __mmplayer_gst_set_pulsesink_property(mmplayer_t *player, MMHandleType attrs) MMPLAYER_FENTER(); MMPLAYER_RETURN_IF_FAIL(player && player->pipeline && player->pipeline->audiobin); - mm_attrs_get_int_by_name(attrs, "sound_stream_index", &stream_id); - mm_attrs_get_string_by_name(attrs, "sound_stream_type", &stream_type); + mm_attrs_get_int_by_name(player->attrs, "sound_stream_index", &stream_id); + mm_attrs_get_string_by_name(player->attrs, "sound_stream_type", &stream_type); if (!stream_type) { LOGE("stream_type is null."); @@ -2526,7 +2518,7 @@ __mmplayer_gst_set_pulsesink_property(mmplayer_t *player, MMHandleType attrs) gst_structure_free(props); } - mm_attrs_get_int_by_name(attrs, "sound_latency_mode", &latency_mode); + mm_attrs_get_int_by_name(player->attrs, "sound_latency_mode", &latency_mode); switch (latency_mode) { case AUDIO_LATENCY_MODE_LOW: @@ -2582,13 +2574,12 @@ __mmplayer_gst_set_openalsink_property(mmplayer_t *player) } static int -__mmplayer_gst_fill_audio_bucket(mmplayer_t *player, GList **bucket) +__mmplayer_gst_make_audio_playback_sink(mmplayer_t *player, GList **bucket) { mmplayer_gst_element_t *audiobin = NULL; - MMHandleType attrs = 0; - GList *element_bucket = NULL; - GstCaps *acaps = NULL; GstPad *sink_pad = NULL; + GstCaps *acaps = NULL; + gint channels = 0; int pitch_control = 0; double pitch_value = 1.0; @@ -2597,17 +2588,17 @@ __mmplayer_gst_fill_audio_bucket(mmplayer_t *player, GList **bucket) player->pipeline->audiobin, MM_ERROR_PLAYER_NOT_INITIALIZED); audiobin = player->pipeline->audiobin; - attrs = MMPLAYER_GET_ATTRS(player); - if (player->build_audio_offload) { /* skip all the audio filters */ - LOGD("create audio offload sink : %s", player->ini.audio_offload_sink_element); - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audio_offload_sink_element, "audiosink", TRUE, player); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "sync", TRUE, NULL); - __mmplayer_add_sink(player, audiobin[MMPLAYER_A_SINK].gst); - goto DONE; - } + LOGD("make element for normal audio playback"); - /* pitch */ + /* audio bin structure for playback. {} means optional. + optional : pitch, audioeq, custom audioeq, openalsink for 360 audio content + + * src - ... - {aconv - pitch} - aconv - rgvolume - resample - volume - + {audioeq} - {custom audioeq} - pulsesink or {aconv - capsfilter - openalsink} + */ + + /* for pitch control */ mm_attrs_multiple_get(player->attrs, NULL, MM_PLAYER_PITCH_CONTROL, &pitch_control, MM_PLAYER_PITCH_VALUE, &pitch_value, @@ -2622,10 +2613,10 @@ __mmplayer_gst_fill_audio_bucket(mmplayer_t *player, GList **bucket) gst_object_unref(factory); /* converter */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV_PITCH, "audioconvert", "audio convert pitch", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV_PITCH, "audioconvert", "audio convert pitch", *bucket, player); /* pitch */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_PITCH, "pitch", "audio pitch", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_PITCH, "pitch", "audio pitch", *bucket, player); g_object_set(G_OBJECT(audiobin[MMPLAYER_A_PITCH].gst), "pitch", (gdouble)pitch_value, NULL); } else { LOGW("there is no pitch element"); @@ -2633,187 +2624,304 @@ __mmplayer_gst_fill_audio_bucket(mmplayer_t *player, GList **bucket) } /* converter */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV, "audioconvert", "audio converter", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV, "audioconvert", "audio converter", *bucket, player); /* replaygain volume */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_RGVOL, "rgvolume", "audio rgvolume", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_RGVOL, "rgvolume", "audio rgvolume", *bucket, player); if (player->sound.rg_enable) g_object_set(G_OBJECT(audiobin[MMPLAYER_A_RGVOL].gst), "enable-rgvolume", TRUE, NULL); else g_object_set(G_OBJECT(audiobin[MMPLAYER_A_RGVOL].gst), "enable-rgvolume", FALSE, NULL); /* resampler */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_RESAMPLER, player->ini.audioresampler_element, "audio resampler", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_RESAMPLER, player->ini.audioresampler_element, "audio resampler", *bucket, player); - if (player->audio_decoded_cb) { /* pcm extraction only, no sound output */ - gchar *dst_format = NULL; - int dst_len = 0; - int dst_samplerate = 0; - int dst_channels = 0; - GstCaps *caps = NULL; - char *caps_str = NULL; + /* for logical volume control */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_VOL, "volume", "volume", *bucket, player); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_VOL].gst), "volume", player->sound.volume, NULL); - /* get conf. values */ - mm_attrs_multiple_get(player->attrs, NULL, - "pcm_audioformat", &dst_format, &dst_len, - "pcm_extraction_samplerate", &dst_samplerate, - "pcm_extraction_channels", &dst_channels, - NULL); + if (player->sound.mute) { + LOGD("mute enabled"); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_VOL].gst), "mute", player->sound.mute, NULL); + } - LOGD("required pcm format - format: %s(%d), samplerate : %d, channel: %d", dst_format, dst_len, dst_samplerate, dst_channels); - if (dst_format == NULL || dst_len == 0 || dst_samplerate == 0 || dst_channels == 0) { - mm_attrs_multiple_get(player->attrs, NULL, - "content_audio_format", &dst_format, &dst_len, /* get string and len */ - "content_audio_samplerate", &dst_samplerate, - "content_audio_channels", &dst_channels, - NULL); + mm_attrs_get_int_by_name(player->attrs, "content_audio_channels", &channels); - LOGD("decoded pcm format - format: %s(%d), samplerate : %d, channel: %d", dst_format, dst_len, dst_samplerate, dst_channels); + /* audio effect element. if audio effect is enabled */ + if ((strcmp(player->ini.audioeffect_element, "")) + && (channels <= 2) + && (player->ini.use_audio_effect_preset || player->ini.use_audio_effect_custom)) { + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_FILTER, player->ini.audioeffect_element, "audio effect filter", *bucket, player); - /* If there is no enough information, set it to platform default value. */ - if (dst_format == NULL || util_convert_audio_pcm_str_to_media_format_mime(dst_format) == MEDIA_FORMAT_MAX) { - LOGD("set platform default format"); - dst_format = DEFAULT_PCM_OUT_FORMAT; + LOGD("audio effect config. bypass = %d, effect type = %d", player->bypass_audio_effect, player->audio_effect_info.effect_type); + + if ((!player->bypass_audio_effect) + && (player->ini.use_audio_effect_preset || player->ini.use_audio_effect_custom)) { + if (player->audio_effect_info.effect_type == MM_AUDIO_EFFECT_TYPE_CUSTOM) { + if (!_mmplayer_audio_effect_custom_apply(player)) + LOGI("apply audio effect(custom) setting success"); } - if (dst_samplerate <= 0) dst_samplerate = DEFAULT_PCM_OUT_SAMPLERATE; - if (dst_channels <= 0) dst_channels = DEFAULT_PCM_OUT_CHANNEL; } - /* capsfilter */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CAPS_DEFAULT, "capsfilter", "audio capsfilter", TRUE, player); - caps = gst_caps_new_simple("audio/x-raw", - "format", G_TYPE_STRING, dst_format, - "rate", G_TYPE_INT, dst_samplerate, - "channels", G_TYPE_INT, dst_channels, - NULL); + if ((strcmp(player->ini.audioeffect_element_custom, "")) + && (player->set_mode.rich_audio)) { + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_FILTER_SEC, player->ini.audioeffect_element_custom, "audio effect filter custom", *bucket, player); + } + } - caps_str = gst_caps_to_string(caps); - LOGD("new caps : %s", caps_str); + /* create audio sink */ + LOGD("spherical %d, channels %d, ambisonic type %d, format %d, order %d", + player->is_content_spherical, channels, player->video360_metadata.ambisonic_type, + player->video360_metadata.ambisonic_format, player->video360_metadata.ambisonic_order); - g_object_set(GST_ELEMENT(audiobin[MMPLAYER_A_CAPS_DEFAULT].gst), "caps", caps, NULL); + /* Note: qtdemux converts audio metadata defaults to openalsink defaults. */ + if (player->is_360_feature_enabled && + player->is_content_spherical && + channels == 4 && + player->video360_metadata.ambisonic_type == MMFILE_AMBISONIC_TYPE_PERIPHONIC && + player->video360_metadata.ambisonic_format == MMFILE_AMBISONIC_FORMAT_AMB && + player->video360_metadata.ambisonic_order == MMFILE_AMBISONIC_ORDER_FOA) { - /* clean */ - gst_caps_unref(caps); - MMPLAYER_FREEIF(caps_str); + strncpy(player->ini.audiosink_element, "openalsink", PLAYER_INI_MAX_STRLEN - 1); - if (player->audio_extract_opt & MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE) { - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_DEINTERLEAVE, "deinterleave", "deinterleave", TRUE, player); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV_BFORMAT, "audioconvert", "audio-converter-bformat", *bucket, player); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_DEINTERLEAVE].gst), "keep-positions", TRUE, NULL); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CAPS_360, "capsfilter", "audio-caps-filter", *bucket, player); + acaps = gst_caps_from_string(SPATIAL_AUDIO_CAPS); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_CAPS_360].gst), "caps", acaps, NULL); + gst_caps_unref(acaps); - /* raw pad handling signal, audiosink will be added after getting signal */ - __mmplayer_add_signal_connection(player, G_OBJECT(audiobin[MMPLAYER_A_DEINTERLEAVE].gst), - MM_PLAYER_SIGNAL_TYPE_AUTOPLUG, "pad-added", G_CALLBACK(__mmplayer_gst_audio_deinterleave_pad_added), (gpointer)player); - } else { - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, "fakesink", "audiosink", TRUE, player); - if (!(player->audio_extract_opt & MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK)) - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "sync", TRUE, NULL); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "signal-handoffs", TRUE, NULL); - - __mmplayer_add_signal_connection(player, - G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), - MM_PLAYER_SIGNAL_TYPE_AUDIOBIN, - "handoff", - G_CALLBACK(__mmplayer_audio_stream_decoded_render_cb), - (gpointer)player); - } + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, "openalsink", "audiosink", *bucket, player); + + player->is_openal_plugin_used = TRUE; } else { - /* normal playback */ - gint channels = 0; + if (player->is_360_feature_enabled && player->is_content_spherical) + LOGW("Audio track isn't of the ambisonic type and can't be played back as a spatial sound."); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audiosink_element, "audiosink", *bucket, player); + } - /* for logical volume control */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_VOL, "volume", "volume", TRUE, player); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_VOL].gst), "volume", player->sound.volume, NULL); + if ((MMPLAYER_IS_RTSP_STREAMING(player)) || + (player->videodec_linked && player->ini.use_system_clock)) { + LOGD("system clock will be used."); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "provide-clock", FALSE, NULL); + } - if (player->sound.mute) { - LOGD("mute enabled"); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_VOL].gst), "mute", player->sound.mute, NULL); - } + if (g_strrstr(player->ini.audiosink_element, "pulsesink")) + __mmplayer_gst_set_pulsesink_property(player); + else if (g_strrstr(player->ini.audiosink_element, "openalsink")) + __mmplayer_gst_set_openalsink_property(player); - mm_attrs_get_int_by_name(player->attrs, "content_audio_channels", &channels); + /* qos on */ + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "qos", TRUE, NULL); /* qos on */ + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "slave-method", GST_AUDIO_BASE_SINK_SLAVE_NONE, NULL); - /* audio effect element. if audio effect is enabled */ - if ((strcmp(player->ini.audioeffect_element, "")) - && (channels <= 2) - && (player->ini.use_audio_effect_preset || player->ini.use_audio_effect_custom)) { - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_FILTER, player->ini.audioeffect_element, "audio effect filter", TRUE, player); + sink_pad = gst_element_get_static_pad(audiobin[MMPLAYER_A_SINK].gst, "sink"); + __mmplayer_add_signal_connection(player, G_OBJECT(sink_pad), MM_PLAYER_SIGNAL_TYPE_AUDIOBIN, + "notify::caps", G_CALLBACK(__mmplayer_gst_caps_notify_cb), (gpointer)player); + gst_object_unref(GST_OBJECT(sink_pad)); - LOGD("audio effect config. bypass = %d, effect type = %d", player->bypass_audio_effect, player->audio_effect_info.effect_type); + __mmplayer_add_sink(player, audiobin[MMPLAYER_A_SINK].gst); - if ((!player->bypass_audio_effect) - && (player->ini.use_audio_effect_preset || player->ini.use_audio_effect_custom)) { - if (player->audio_effect_info.effect_type == MM_AUDIO_EFFECT_TYPE_CUSTOM) { - if (!_mmplayer_audio_effect_custom_apply(player)) - LOGI("apply audio effect(custom) setting success"); - } - } + MMPLAYER_FLEAVE(); + return MM_ERROR_NONE; - if ((strcmp(player->ini.audioeffect_element_custom, "")) - && (player->set_mode.rich_audio)) - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_FILTER_SEC, player->ini.audioeffect_element_custom, "audio effect filter custom", TRUE, player); - } +ERROR: /* MMPLAYER_CREATE_ELEMENT */ + MMPLAYER_FLEAVE(); + return MM_ERROR_PLAYER_INTERNAL; +} - /* create audio sink */ - LOGD("360 spherical %d, channels %d, ambisonic type %d, format %d, order %d", - player->is_content_spherical, channels, player->video360_metadata.ambisonic_type, - player->video360_metadata.ambisonic_format, player->video360_metadata.ambisonic_order); +static int +__mmplayer_gst_make_audio_extract_sink(mmplayer_t *player, GList **bucket) +{ + mmplayer_gst_element_t *audiobin = NULL; + enum audio_element_id extract_sink_id = MMPLAYER_A_SINK; - /* Note: qtdemux converts audio metadata defaults to openalsink defaults. */ - if (player->is_360_feature_enabled && - player->is_content_spherical && - channels == 4 && - player->video360_metadata.ambisonic_type == MMFILE_AMBISONIC_TYPE_PERIPHONIC && - player->video360_metadata.ambisonic_format == MMFILE_AMBISONIC_FORMAT_AMB && - player->video360_metadata.ambisonic_order == MMFILE_AMBISONIC_ORDER_FOA) { + gchar *dst_format = NULL; + int dst_len = 0; + int dst_samplerate = 0; + int dst_channels = 0; + GstCaps *caps = NULL; + char *caps_str = NULL; + + MMPLAYER_FENTER(); + MMPLAYER_RETURN_VAL_IF_FAIL(player && player->pipeline && + player->pipeline->audiobin, MM_ERROR_PLAYER_NOT_INITIALIZED); - strncpy(player->ini.audiosink_element, "openalsink", PLAYER_INI_MAX_STRLEN - 1); + audiobin = player->pipeline->audiobin; - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV_BFORMAT, "audioconvert", "audio-converter-bformat", TRUE, player); + LOGD("make element for audio extract, option = 0x%X", player->audio_extract_opt); - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CAPS_360, "capsfilter", "audio-caps-filter", TRUE, player); - acaps = gst_caps_from_string(SPATIAL_AUDIO_CAPS); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_CAPS_360].gst), "caps", acaps, NULL); - gst_caps_unref(acaps); + /* audio bin structure according to the mmplayer_audio_extract_opt_e. - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, "openalsink", "audiosink", TRUE, player); + [case 1] extract interleave audio pcm without playback + : MM_PLAYER_AUDIO_EXTRACT_DEFAULT (sync) + MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK (non sync) - player->is_openal_plugin_used = TRUE; - } else { - if (player->is_360_feature_enabled && player->is_content_spherical) - LOGW("Audio track isn't of the ambisonic type and can't be played back as a spatial sound."); - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audiosink_element, "audiosink", TRUE, player); - } + * src - ... - aconv - resample - capsfilter - fakesink (sync or not) - if ((MMPLAYER_IS_RTSP_STREAMING(player)) || - (player->videodec_linked && player->ini.use_system_clock)) { - LOGD("system clock will be used."); - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "provide-clock", FALSE, NULL); + [case 2] deinterleave for each channel without playback + : MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE (sync) + MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_AND_DEINTERLEAVE (non sync) + + * src - ... - aconv - resample - capsfilter - deinterleave - fakesink (sync or not) + - fakesink (sync or not) + - ... (sync or not) + + [case 3] [case 1(sync only)] + playback + : MM_PLAYER_AUDIO_EXTRACT_WITH_PLAYBACK + + * src - ... - tee - queue1 - playback path + - queue2 - [case1 pipeline with sync] + + [case 4] [case 2(sync only)] + playback + : MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE_WITH_PLAYBACK + + * src - ... - tee - queue1 - playback path + - queue2 - [case2 pipeline with sync] + + */ + + /* 1. create tee and playback path + 'tee' should be added at first to copy the decoded stream + */ + if (player->audio_extract_opt & MM_PLAYER_AUDIO_EXTRACT_WITH_PLAYBACK) { + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_TEE, "tee", "audio-tee", *bucket, player); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_TEE].gst), "num-src-pads", 2, NULL); + + /* tee - path 1 : for playback path */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_TEE_Q1, "queue", "audio-tee-queue1", *bucket, player); + __mmplayer_gst_make_audio_playback_sink(player, bucket); + + /* tee - path 2 : for extract path */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_TEE_Q2, "queue", "audio-tee-queue2", *bucket, player); + extract_sink_id = MMPLAYER_A_EXTRACT_SINK; /* there is another playback sink */ + } + + /* if there is tee, 'tee - path 2' is linked here */ + /* converter */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_EXTRACT_CONV, "audioconvert", "audio-ext-conv", *bucket, player); + + /* resampler */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_EXTRACT_RESAMPLER, player->ini.audioresampler_element, "audio-ext-resampler", *bucket, player); + + /* 2. decide the extract pcm format */ + mm_attrs_multiple_get(player->attrs, NULL, + "pcm_audioformat", &dst_format, &dst_len, + "pcm_extraction_samplerate", &dst_samplerate, + "pcm_extraction_channels", &dst_channels, + NULL); + + LOGD("required extract pcm format - format: %s(%d), samplerate : %d, channel: %d", + dst_format, dst_len, dst_samplerate, dst_channels); + + if (dst_format == NULL || dst_len == 0 || dst_samplerate == 0 || dst_channels == 0) { + mm_attrs_multiple_get(player->attrs, NULL, + "content_audio_format", &dst_format, &dst_len, /* get string and len */ + "content_audio_samplerate", &dst_samplerate, + "content_audio_channels", &dst_channels, + NULL); + + LOGD("apply the decoded pcm format - format: %s(%d), samplerate : %d, channel: %d", + dst_format, dst_len, dst_samplerate, dst_channels); + + /* If there is no enough information, set it to platform default value. */ + if (dst_format == NULL || util_convert_audio_pcm_str_to_media_format_mime(dst_format) == MEDIA_FORMAT_MAX) { + LOGD("set platform default format"); + dst_format = DEFAULT_PCM_OUT_FORMAT; } + if (dst_samplerate <= 0) dst_samplerate = DEFAULT_PCM_OUT_SAMPLERATE; + if (dst_channels <= 0) dst_channels = DEFAULT_PCM_OUT_CHANNEL; + } - if (g_strrstr(player->ini.audiosink_element, "pulsesink")) - __mmplayer_gst_set_pulsesink_property(player, attrs); - else if (g_strrstr(player->ini.audiosink_element, "openalsink")) - __mmplayer_gst_set_openalsink_property(player); + /* 3. create capsfilter */ + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_EXTRACT_CAPS, "capsfilter", "audio-ext-caps", *bucket, player); + caps = gst_caps_new_simple("audio/x-raw", + "format", G_TYPE_STRING, dst_format, + "rate", G_TYPE_INT, dst_samplerate, + "channels", G_TYPE_INT, dst_channels, + NULL); - /* qos on */ - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "qos", TRUE, NULL); /* qos on */ - g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "slave-method", GST_AUDIO_BASE_SINK_SLAVE_NONE, NULL); + caps_str = gst_caps_to_string(caps); + LOGD("new caps : %s", caps_str); - sink_pad = gst_element_get_static_pad(audiobin[MMPLAYER_A_SINK].gst, "sink"); - __mmplayer_add_signal_connection(player, G_OBJECT(sink_pad), MM_PLAYER_SIGNAL_TYPE_AUDIOBIN, - "notify::caps", G_CALLBACK(__mmplayer_gst_caps_notify_cb), (gpointer)player); - gst_object_unref(GST_OBJECT(sink_pad)); + g_object_set(GST_ELEMENT(audiobin[MMPLAYER_A_EXTRACT_CAPS].gst), "caps", caps, NULL); + + /* clean */ + gst_caps_unref(caps); + MMPLAYER_FREEIF(caps_str); + + /* 4-1. create deinterleave to extract pcm for each channel */ + if (player->audio_extract_opt & MM_PLAYER_AUDIO_EXTRACT_DEINTERLEAVE) { + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_EXTRACT_DEINTERLEAVE, "deinterleave", "deinterleave", *bucket, player); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_EXTRACT_DEINTERLEAVE].gst), "keep-positions", TRUE, NULL); + + /* audiosink will be added after getting signal for each channel */ + __mmplayer_add_signal_connection(player, G_OBJECT(audiobin[MMPLAYER_A_EXTRACT_DEINTERLEAVE].gst), + MM_PLAYER_SIGNAL_TYPE_AUTOPLUG, "pad-added", G_CALLBACK(__mmplayer_gst_audio_deinterleave_pad_added), (gpointer)player); + } else { + /* 4-2. create fakesink to extract interlevaed pcm */ + LOGD("add audio fakesink for interleaved audio"); + MMPLAYER_CREATE_ELEMENT(audiobin, extract_sink_id, "fakesink", "fakeaudiosink", *bucket, player); + if (!(player->audio_extract_opt & MM_PLAYER_AUDIO_EXTRACT_NO_SYNC_WITH_CLOCK)) + g_object_set(G_OBJECT(audiobin[extract_sink_id].gst), "sync", TRUE, NULL); + g_object_set(G_OBJECT(audiobin[extract_sink_id].gst), "signal-handoffs", TRUE, NULL); + __mmplayer_add_signal_connection(player, + G_OBJECT(audiobin[extract_sink_id].gst), + MM_PLAYER_SIGNAL_TYPE_AUDIOBIN, + "handoff", + G_CALLBACK(__mmplayer_audio_stream_decoded_render_cb), + (gpointer)player); + + __mmplayer_add_sink(player, audiobin[extract_sink_id].gst); + } + + MMPLAYER_FLEAVE(); + return MM_ERROR_NONE; + +ERROR: /* MMPLAYER_CREATE_ELEMENT */ + MMPLAYER_FLEAVE(); + return MM_ERROR_PLAYER_INTERNAL; +} + +static int +__mmplayer_gst_make_audio_bin_element(mmplayer_t *player, GList **bucket) +{ + int ret = MM_ERROR_NONE; + mmplayer_gst_element_t *audiobin = NULL; + GList *element_bucket = NULL; + + MMPLAYER_FENTER(); + MMPLAYER_RETURN_VAL_IF_FAIL(player && player->pipeline && + player->pipeline->audiobin, MM_ERROR_PLAYER_NOT_INITIALIZED); + + audiobin = player->pipeline->audiobin; + + if (player->build_audio_offload) { /* skip all the audio filters */ + LOGD("create audio offload sink : %s", player->ini.audio_offload_sink_element); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audio_offload_sink_element, "audiosink", element_bucket, player); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "sync", TRUE, NULL); __mmplayer_add_sink(player, audiobin[MMPLAYER_A_SINK].gst); + goto DONE; } + /* FIXME: need to mention the supportable condition at API reference */ + if (player->audio_decoded_cb && (!MMPLAYER_IS_RTSP_STREAMING(player))) + ret = __mmplayer_gst_make_audio_extract_sink(player, &element_bucket); + else + ret = __mmplayer_gst_make_audio_playback_sink(player, &element_bucket); + + if (ret != MM_ERROR_NONE) + goto ERROR; DONE: + LOGD("success to make audio bin element"); *bucket = element_bucket; MMPLAYER_FLEAVE(); return MM_ERROR_NONE; ERROR: + LOGE("failed to make audio bin element"); g_list_free(element_bucket); *bucket = NULL; @@ -2853,7 +2961,7 @@ __mmplayer_gst_create_audio_sink_bin(mmplayer_t *player) player->pipeline->audiobin = audiobin; /* create audio filters and audiosink */ - if (__mmplayer_gst_fill_audio_bucket(player, &element_bucket) != MM_ERROR_NONE) + if (__mmplayer_gst_make_audio_bin_element(player, &element_bucket) != MM_ERROR_NONE) goto ERROR; /* adding created elements to bin */ @@ -3250,7 +3358,7 @@ __mmplayer_gst_create_video_filters(mmplayer_t *player, MMDisplaySurfaceType sur /* create video360 filter */ if (player->is_360_feature_enabled && player->is_content_spherical) { LOGD("create video360 element"); - MMPLAYER_CREATE_ELEMENT(player->pipeline->videobin, MMPLAYER_V_360, "video360", "video-360", TRUE, player); + MMPLAYER_CREATE_ELEMENT(player->pipeline->videobin, MMPLAYER_V_360, "video360", "video-360", element_bucket, player); __mmplayer_gst_set_video360_property(player); goto EXIT; } @@ -3263,7 +3371,7 @@ __mmplayer_gst_create_video_filters(mmplayer_t *player, MMDisplaySurfaceType sur /* in case of sw codec & overlay surface type, except 360 playback. * if libav video decoder is selected, videoconvert is required to render the shm wl-buffer which support RGB only via tizenwlsink. */ LOGD("create video converter: %s", video_csc); - MMPLAYER_CREATE_ELEMENT(player->pipeline->videobin, MMPLAYER_V_CONV, video_csc, "video converter", TRUE, player); + MMPLAYER_CREATE_ELEMENT(player->pipeline->videobin, MMPLAYER_V_CONV, video_csc, "video converter", element_bucket, player); EXIT: *bucket = element_bucket; @@ -3417,7 +3525,7 @@ __mmplayer_gst_create_video_sink_bin(mmplayer_t *player, GstCaps *caps, MMDispla goto ERROR; videosink_factory_name = __mmplayer_get_videosink_factory_name(player, surface_type); - MMPLAYER_CREATE_ELEMENT(videobin, MMPLAYER_V_SINK, videosink_factory_name, "videosink", TRUE, player); + MMPLAYER_CREATE_ELEMENT(videobin, MMPLAYER_V_SINK, videosink_factory_name, "videosink", element_bucket, player); /* additional setting for sink plug-in */ if (__mmplayer_gst_set_videosink_property(player, surface_type) != MM_ERROR_NONE) { @@ -3491,13 +3599,13 @@ __mmplayer_gst_create_plain_text_elements(mmplayer_t *player) GList *element_bucket = NULL; mmplayer_gst_element_t *textbin = player->pipeline->textbin; - MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_QUEUE, "queue", "text_queue", TRUE, player); - MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_IDENTITY, "identity", "text_identity", TRUE, player); + MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_QUEUE, "queue", "text_queue", element_bucket, player); + MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_IDENTITY, "identity", "text_identity", element_bucket, player); g_object_set(G_OBJECT(textbin[MMPLAYER_T_IDENTITY].gst), "signal-handoffs", FALSE, NULL); - MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_FAKE_SINK, "fakesink", "text_fakesink", TRUE, player); + MMPLAYER_CREATE_ELEMENT(textbin, MMPLAYER_T_FAKE_SINK, "fakesink", "text_fakesink", element_bucket, player); __mmplayer_add_signal_connection(player, G_OBJECT(textbin[MMPLAYER_T_FAKE_SINK].gst), MM_PLAYER_SIGNAL_TYPE_TEXTBIN, @@ -8890,17 +8998,25 @@ __mmplayer_update_audio_attrs(mmplayer_t *player, MMHandleType attrs) GstPad *pad = NULL; gint samplerate = 0, channels = 0; GstStructure *p = NULL; + GstElement *aconv = NULL; LOGD("try to update audio attrs"); MMPLAYER_RETURN_VAL_IF_FAIL(player->pipeline->audiobin, FALSE); - MMPLAYER_RETURN_VAL_IF_FAIL(player->pipeline->audiobin[MMPLAYER_A_SINK].gst, FALSE); - pad = gst_element_get_static_pad( - player->pipeline->audiobin[MMPLAYER_A_CONV].gst, "sink"); + if (player->pipeline->audiobin[MMPLAYER_A_CONV].gst) { + aconv = player->pipeline->audiobin[MMPLAYER_A_CONV].gst; + } else if (player->pipeline->audiobin[MMPLAYER_A_EXTRACT_CONV].gst) { + aconv = player->pipeline->audiobin[MMPLAYER_A_EXTRACT_CONV].gst; + } else { + LOGE("there is no audio converter"); + return FALSE; + } + + pad = gst_element_get_static_pad(aconv, "sink"); if (!pad) { - LOGW("failed to get pad from audiosink"); + LOGW("failed to get pad from audio converter"); return FALSE; } diff --git a/src/mm_player_utils.c b/src/mm_player_utils.c index a9d4f90..282b2b4 100644 --- a/src/mm_player_utils.c +++ b/src/mm_player_utils.c @@ -196,6 +196,8 @@ __mmplayer_dump_pipeline_state(mmplayer_t *player) player->pipeline->mainbin, FALSE); + MMPLAYER_GENERATE_DOT_IF_ENABLED(player, "pipeline-status-error-dump"); + iter = gst_bin_iterate_recurse(GST_BIN(player->pipeline->mainbin[MMPLAYER_M_PIPE].gst)); if (iter != NULL) { -- 2.7.4