From: Andriy Martynets Date: Wed, 27 Sep 2017 13:12:47 +0000 (+0300) Subject: [0.6.87] Video360 and openalsink elements added, video360 internal API implementation... X-Git-Tag: submit/tizen_4.0/20180109.072718^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=cac209eb530d5f42877fb0eea7bf56ce5f9a540d;p=platform%2Fcore%2Fmultimedia%2Flibmm-player.git [0.6.87] Video360 and openalsink elements added, video360 internal API implementation added Change-Id: I86e9cf118d4193369855755a52de1cc6524a0cde (cherry picked from commit 7c97d85b1367d0d968bbc4ab69e3ac0ef298bbf1) (cherry picked from commit 020babc257f8cc1f394490fcfa29fbf923d7a833) --- diff --git a/configure.ac b/configure.ac index 83334ce..850ff61 100644 --- a/configure.ac +++ b/configure.ac @@ -110,6 +110,11 @@ PKG_CHECK_MODULES(SYSTEMINFO, capi-system-info) AC_SUBST(SYSTEMINFO_CFLAGS) AC_SUBST(SYSTEMINFO_LIBS) +# for Spatial audio +PKG_CHECK_MODULES(MEDIASOUNDMGR,capi-media-sound-manager) +AC_SUBST(MEDIASOUNDMGR_CFLAGS) +AC_SUBST(MEDIASOUNDMGR_LIBS) + AC_ARG_ENABLE(sdk, AC_HELP_STRING([--enable-sdk], [sdk build]), [ case "${enableval}" in diff --git a/packaging/libmm-player.spec b/packaging/libmm-player.spec index 978c564..13e446e 100644 --- a/packaging/libmm-player.spec +++ b/packaging/libmm-player.spec @@ -1,6 +1,6 @@ Name: libmm-player Summary: Multimedia Framework Player Library -Version: 0.6.86 +Version: 0.6.87 Release: 0 Group: Multimedia/Libraries License: Apache-2.0 @@ -25,6 +25,7 @@ BuildRequires: pkgconfig(murphy-glib) BuildRequires: pkgconfig(capi-system-info) BuildRequires: pkgconfig(storage) BuildRequires: pkgconfig(libtzplatform-config) +BuildRequires: pkgconfig(capi-media-sound-manager) %description Multimedia Framework Player Library files. diff --git a/src/Makefile.am b/src/Makefile.am index 698793c..ac49091 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,7 +19,8 @@ libmmfplayer_la_SOURCES = mm_player.c \ mm_player_es.c \ mm_player_common_priv.c \ mm_player_attrs.c \ - mm_player_ini.c + mm_player_ini.c \ + mm_player_360.c libmmfplayer_la_CFLAGS = -I$(srcdir)/include \ $(MMCOMMON_CFLAGS) \ @@ -38,6 +39,7 @@ libmmfplayer_la_CFLAGS = -I$(srcdir)/include \ $(SYSTEMINFO_CFLAGS) \ $(STORAGE_CFLAGS) \ $(TZPLATFORM_CONFIG_CFLAGS) \ + $(MEDIASOUNDMGR_CFLAGS) \ -Werror -Wno-deprecated -Wno-deprecated-declarations -Wno-cpp noinst_HEADERS = include/mm_player_utils.h \ @@ -67,10 +69,10 @@ libmmfplayer_la_LIBADD = $(GST_LIBS) \ $(MEDIAPACKET_LIBS) \ $(DLOG_LIBS) \ $(UTILX_LIBS) \ - $(SYSTEMINFO_LIBS) + $(SYSTEMINFO_LIBS) \ $(STORAGE_LIBS) \ - $(TZPLATFORM_CONFIG_LIBS) - + $(TZPLATFORM_CONFIG_LIBS) \ + $(MEDIASOUNDMGR_LIBS) if IS_SDK libmmfplayer_la_CFLAGS += -DIS_SDK diff --git a/src/include/mm_player_priv.h b/src/include/mm_player_priv.h index f4b6158..2303b4e 100644 --- a/src/include/mm_player_priv.h +++ b/src/include/mm_player_priv.h @@ -80,6 +80,8 @@ #define MM_MAX_STRING_LENGTH 4000 /* have to be less than MUSE_MSG_MAX_LENGTH */ +#define VIDEO360_MAX_ZOOM 10.0f + /*--------------------------------------------------------------------------- | GLOBAL CONSTANT DEFINITIONS: | ---------------------------------------------------------------------------*/ @@ -227,6 +229,8 @@ enum AudioElementID { MMPLAYER_A_FILTER, MMPLAYER_A_FILTER_SEC, MMPLAYER_A_CAPS_DEFAULT, + MMPLAYER_A_CONV_BFORMAT, + MMPLAYER_A_CAPS_360, MMPLAYER_A_SINK, MMPLAYER_A_RESAMPLER, MMPLAYER_A_DEINTERLEAVE, @@ -241,6 +245,7 @@ enum VideoElementID { MMPLAYER_V_SCALE, MMPLAYER_V_CAPS, MMPLAYER_V_SINK, + MMPLAYER_V_360, MMPLAYER_V_NUM }; @@ -335,6 +340,49 @@ typedef enum { MMPLAYER_PATH_MAX } MMPlayerPathType; +/* Video360 related enums. These are duplication from video360.h from + * gst-plugins-tizen. + * */ +typedef enum { + VIDEO360_MODE_UNKNOWN = -1, + VIDEO360_MODE_MONOSCOPIC = 0, + VIDEO360_MODE_STEREOSCOPIC_TOP_BOTTOM = 1, + VIDEO360_MODE_STEREOSCOPIC_LEFT_RIGHT = 2, + VIDEO360_MODE_STEREOSCOPIC_STEREO_MESH = 3, + VIDEO360_MODE_LAST_ITEM +} mm_player_stereo_mode_e; + +typedef enum { + VIDEO360_PROJECTION_TYPE_UNKNOWN = -1, + VIDEO360_PROJECTION_TYPE_EQUIRECTANGULAR = 0, + VIDEO360_PROJECTION_TYPE_CUBEMAP = 1, + VIDEO360_PROJECTION_TYPE_MESH = 2, + VIDEO360_PROJECTION_TYPE_LAST_ITEM +} mm_player_projection_type_e; + +/* Spatial audio related enums. These are duplication from mm_file.h from + * libmm-fileinfo. + * */ +typedef enum { + MMFILE_AMBISONIC_TYPE_UNKNOWN = 0, + MMFILE_AMBISONIC_TYPE_PERIPHONIC = 1, /**< To comply with Google's Spatial Audio RFC*/ + MMFILE_AMBISONIC_TYPE_NON_PERIPHONIC = 2, +} MMFILE_AMBISONIC_TYPE; + +typedef enum { + MMFILE_AMBISONIC_FORMAT_UNKNOWN = 0, + MMFILE_AMBISONIC_FORMAT_AMBIX = 1, /**< AMBIX (Channel sequence: ACN, Normalization: SN3D) */ + MMFILE_AMBISONIC_FORMAT_AMB = 2, /**< .AMB, Tetraproc (Channel sequence: FuMa, Normalization: FuMa) */ + MMFILE_AMBISONIC_FORMAT_UA = 3, /**< Universal Ambisonics (Channel sequence: SID, Normalization: N3D) */ +} MMFILE_AMBISONIC_FORMAT; + +typedef enum { + MMFILE_AMBISONIC_ORDER_UNKNOWN = 0, + MMFILE_AMBISONIC_ORDER_FOA = 1, /**< First order Ambisonics */ + MMFILE_AMBISONIC_ORDER_SOA = 2, /**< Second order Ambisonics */ + MMFILE_AMBISONIC_ORDER_TOA = 3, /**< Third order Ambisonics */ +} MMFILE_AMBISONIC_ORDER; + /*--------------------------------------------------------------------------- | GLOBAL DATA TYPE DEFINITIONS: | ---------------------------------------------------------------------------*/ @@ -458,6 +506,30 @@ typedef struct { VariantData limit; } MMAdaptiveVariantInfo; +typedef struct { + int is_spherical; + int is_stitched; + char *stitching_software; + char *projection_type_string; + char *stereo_mode_string; + mm_player_projection_type_e projection_type; + mm_player_stereo_mode_e stereo_mode; + int source_count; + int init_view_heading; + int init_view_pitch; + int init_view_roll; + int timestamp; + int full_pano_width_pixels; + int full_pano_height_pixels; + int cropped_area_image_width; + int cropped_area_image_height; + int cropped_area_left; + int cropped_area_top; + int ambisonic_type; + int ambisonic_format; + int ambisonic_order; +} mm_player_spherical_metadata_t; + typedef struct { /* STATE */ int state; // player current state @@ -769,6 +841,21 @@ typedef struct { int pcm_channel; MMAdaptiveVariantInfo adaptive_info; + + /* Video360 related stuff + * */ + bool is_content_spherical; + mm_player_spherical_metadata_t video360_metadata; + bool is_openal_plugin_used; + bool is_video360_plugin_used; + /* User settable values */ + bool is_video360_enabled; + float video360_yaw_radians; + float video360_pitch_radians; + float video360_zoom; + int video360_horizontal_fov; + int video360_vertical_fov; + } mm_player_t; typedef struct { diff --git a/src/mm_player_360.c b/src/mm_player_360.c new file mode 100644 index 0000000..738ed03 --- /dev/null +++ b/src/mm_player_360.c @@ -0,0 +1,182 @@ +/* + * libmm-player + * + * Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "mm_player_priv.h" +#include "mm_player_360.h" +#include +#include +#include + +int _mmplayer_360_set_enable(MMHandleType player, bool enable) +{ + ((mm_player_t*) player)->is_video360_enabled = enable; + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + /* We will get here if player is pending ready or ready and above */ + g_object_set(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "passthrough", !enable, NULL); + return MM_ERROR_NONE; + } + + return ((mm_player_t*) player)->state < MM_PLAYER_STATE_READY ? MM_ERROR_NONE : MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_is_enabled(MMHandleType player, bool *enabled) +{ + if (((mm_player_t*) player)->state < MM_PLAYER_STATE_READY) { + *enabled = ((mm_player_t*) player)->is_video360_enabled; + return MM_ERROR_NONE; + } + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + g_object_get(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "passthrough", enabled, NULL); + *enabled = !(*enabled); + return MM_ERROR_NONE; + } + + return MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_set_direction_of_view(MMHandleType player, float yaw, float pitch) +{ + if (yaw > M_PI || yaw < -M_PI || pitch > M_PI_2 || pitch < -M_PI_2) + return MM_ERROR_INVALID_ARGUMENT; + + ((mm_player_t*) player)->video360_yaw_radians = yaw; + ((mm_player_t*) player)->video360_pitch_radians = pitch; + + if (((mm_player_t*) player)->is_content_spherical) { + /* We will get here if player is pending ready or ready and above */ + if (((mm_player_t*) player)->is_video360_plugin_used) { + g_object_set(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "pose-yaw", (int) (yaw * 180.0f / M_PI), + "pose-pitch", (int) (pitch * 180.0f / M_PI), NULL); + } + if (((mm_player_t*) player)->is_openal_plugin_used) { + g_object_set(G_OBJECT(((mm_player_t*) player)->pipeline->audiobin[MMPLAYER_A_SINK].gst), + "source-orientation-y", (int) (yaw * 180.0f / M_PI), + "source-orientation-x", (int) (pitch * 180.0f / M_PI), NULL); + } + return MM_ERROR_NONE; + } + + return ((mm_player_t*) player)->state < MM_PLAYER_STATE_READY ? MM_ERROR_NONE : MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_get_direction_of_view(MMHandleType player, float *yaw, float *pitch) +{ + int yaw_degrees, pitch_degrees; + + if (((mm_player_t*) player)->state < MM_PLAYER_STATE_READY) { + *yaw = ((mm_player_t*) player)->video360_yaw_radians; + *pitch = ((mm_player_t*) player)->video360_pitch_radians; + return MM_ERROR_NONE; + } + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + g_object_get(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "pose-yaw", &yaw_degrees, "pose-pitch", &pitch_degrees, NULL); + *yaw = M_PI * yaw_degrees / 180.0f; + *pitch = M_PI * pitch_degrees / 180.0f; + return MM_ERROR_NONE; + } + + return MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_set_zoom(MMHandleType player, float level) +{ + if (level < 1.0f || level > VIDEO360_MAX_ZOOM) + return MM_ERROR_INVALID_ARGUMENT; + + ((mm_player_t*) player)->video360_zoom = level; + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + /* We will get here if player is pending ready or ready and above */ + g_object_set(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "zoom", 1.0f / level, NULL); + return MM_ERROR_NONE; + } + + return ((mm_player_t*) player)->state < MM_PLAYER_STATE_READY ? MM_ERROR_NONE : MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_get_zoom(MMHandleType player, float *level) +{ + float current_zoom; + + if (((mm_player_t*) player)->state < MM_PLAYER_STATE_READY) { + *level = ((mm_player_t*) player)->video360_zoom; + return MM_ERROR_NONE; + } + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + g_object_get(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "zoom", ¤t_zoom, NULL); + *level = 1.0f / current_zoom; + return MM_ERROR_NONE; + } + + return MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_set_field_of_view(MMHandleType player, int horizontal_degrees, int vertical_degrees) +{ + if (horizontal_degrees < 1 || horizontal_degrees > 360 || vertical_degrees < 1 || vertical_degrees > 180) + return MM_ERROR_INVALID_ARGUMENT; + + ((mm_player_t*) player)->video360_horizontal_fov = horizontal_degrees; + ((mm_player_t*) player)->video360_vertical_fov = vertical_degrees; + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + /* We will get here if player is pending ready or ready and above */ + g_object_set(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "horizontal-fov", horizontal_degrees, "vertical-fov", vertical_degrees, NULL); + return MM_ERROR_NONE; + } + + return ((mm_player_t*) player)->state < MM_PLAYER_STATE_READY ? MM_ERROR_NONE : MM_ERROR_PLAYER_INTERNAL; +} + +int _mmplayer_360_get_field_of_view(MMHandleType player, int *horizontal_degrees, int *vertical_degrees) +{ + if (((mm_player_t*) player)->state < MM_PLAYER_STATE_READY) { + *horizontal_degrees = ((mm_player_t*) player)->video360_horizontal_fov; + *vertical_degrees = ((mm_player_t*) player)->video360_vertical_fov; + return MM_ERROR_NONE; + } + + if (((mm_player_t*) player)->is_content_spherical && + ((mm_player_t*) player)->is_video360_plugin_used) { + g_object_get(G_OBJECT(((mm_player_t*) player)->pipeline->videobin[MMPLAYER_V_360].gst), + "horizontal-fov", horizontal_degrees, "vertical-fov", vertical_degrees, NULL); + return MM_ERROR_NONE; + } + + return MM_ERROR_PLAYER_INTERNAL; +} diff --git a/src/mm_player_attrs.c b/src/mm_player_attrs.c index bddd16f..c214dbf 100644 --- a/src/mm_player_attrs.c +++ b/src/mm_player_attrs.c @@ -1097,6 +1097,15 @@ _mmplayer_construct_attribute(MMHandleType handle) FALSE, TRUE }, + { + "content_video_is_spherical", + MM_ATTRS_TYPE_INT, + MM_ATTRS_FLAG_RW, + (void *) -1, + MM_ATTRS_VALID_TYPE_INT_RANGE, + -1, + MMPLAYER_MAX_INT + }, { "video_codec_type", MM_ATTRS_TYPE_INT, @@ -1106,7 +1115,6 @@ _mmplayer_construct_attribute(MMHandleType handle) MM_PLAYER_VIDEO_CODEC_TYPE_DEFAULT, MM_PLAYER_VIDEO_CODEC_TYPE_SW }, - }; num_of_attrs = ARRAY_SIZE(player_attrs); diff --git a/src/mm_player_priv.c b/src/mm_player_priv.c index 1f51331..20b03d9 100644 --- a/src/mm_player_priv.c +++ b/src/mm_player_priv.c @@ -49,8 +49,10 @@ #include "mm_player_capture.h" #include "mm_player_utils.h" #include "mm_player_tracks.h" +#include "mm_player_360.h" #include +#include /*=========================================================================================== | | @@ -101,6 +103,8 @@ #define PLAYER_BUS_MSG_DEFAULT_TIMEOUT 500 /* bus msg wait timeout */ #define PLAYER_BUS_MSG_PREPARE_TIMEOUT 100 +#define SPATIAL_AUDIO_CAPS "audio/x-raw,format=S16LE,channels=4" + /*--------------------------------------------------------------------------- | LOCAL CONSTANT DEFINITIONS: | ---------------------------------------------------------------------------*/ @@ -116,6 +120,7 @@ /*--------------------------------------------------------------------------- | LOCAL VARIABLE DEFINITIONS: | ---------------------------------------------------------------------------*/ +static sound_stream_info_h stream_info; /*--------------------------------------------------------------------------- | LOCAL FUNCTION PROTOTYPES: | @@ -242,6 +247,7 @@ static void __mmplayer_gst_caps_notify_cb(GstPad * pad, GParamSpec * unused, gp static void __mmplayer_audio_stream_clear_buffer(mm_player_t* player, gboolean send_all); static void __mmplayer_audio_stream_send_data(mm_player_t* player, mm_player_audio_stream_buff_t *a_buffer); static void __mmplayer_initialize_storage_info(mm_player_t* player, MMPlayerPathType path_type); +static void __mmplayer_get_metadata_360_from_tags(GstTagList *tags, mm_player_spherical_metadata_t *metadata); /*=========================================================================================== | | @@ -1871,6 +1877,36 @@ __mmplayer_gst_handle_duration(mm_player_t* player, GstMessage* msg) return TRUE; } +static void __mmplayer_get_metadata_360_from_tags(GstTagList *tags, + mm_player_spherical_metadata_t *metadata) { + gst_tag_list_get_int(tags, "is_spherical", &metadata->is_spherical); + gst_tag_list_get_int(tags, "is_stitched", &metadata->is_stitched); + gst_tag_list_get_string(tags, "stitching_software", + &metadata->stitching_software); + gst_tag_list_get_string(tags, "projection_type", + &metadata->projection_type_string); + gst_tag_list_get_string(tags, "stereo_mode", &metadata->stereo_mode_string); + gst_tag_list_get_int(tags, "source_count", &metadata->source_count); + gst_tag_list_get_int(tags, "init_view_heading", + &metadata->init_view_heading); + gst_tag_list_get_int(tags, "init_view_pitch", &metadata->init_view_pitch); + gst_tag_list_get_int(tags, "init_view_roll", &metadata->init_view_roll); + gst_tag_list_get_int(tags, "timestamp", &metadata->timestamp); + gst_tag_list_get_int(tags, "full_pano_width_pixels", + &metadata->full_pano_width_pixels); + gst_tag_list_get_int(tags, "full_pano_height_pixels", + &metadata->full_pano_height_pixels); + gst_tag_list_get_int(tags, "cropped_area_image_width", + &metadata->cropped_area_image_width); + gst_tag_list_get_int(tags, "cropped_area_image_height", + &metadata->cropped_area_image_height); + gst_tag_list_get_int(tags, "cropped_area_left", + &metadata->cropped_area_left); + gst_tag_list_get_int(tags, "cropped_area_top", &metadata->cropped_area_top); + gst_tag_list_get_int(tags, "ambisonic_type", &metadata->ambisonic_type); + gst_tag_list_get_int(tags, "ambisonic_format", &metadata->ambisonic_format); + gst_tag_list_get_int(tags, "ambisonic_order", &metadata->ambisonic_order); +} static gboolean __mmplayer_gst_extract_tag_from_msg(mm_player_t* player, GstMessage* msg) @@ -2071,6 +2107,37 @@ if (gst_tag_list_get_double(tag_list, gsttag, &v_double)) {\ /* MMPLAYER_UPDATE_TAG_DOUBLE(GST_TAG_BEATS_PER_MINUTE, ?, ?); */ MMPLAYER_UPDATE_TAG_STRING(GST_TAG_IMAGE_ORIENTATION, attrs, "content_video_orientation"); + if (strstr(GST_OBJECT_NAME(msg->src), "demux")) { + if (player->video360_metadata.is_spherical == -1) { + __mmplayer_get_metadata_360_from_tags(tag_list, &player->video360_metadata); + mm_attrs_set_int_by_name(attrs, "content_video_is_spherical", + player->video360_metadata.is_spherical); + player->is_content_spherical = + player->video360_metadata.is_spherical == 1 ? TRUE : FALSE; + if (player->video360_metadata.projection_type_string) { + if (!strcmp(player->video360_metadata.projection_type_string, "equirectangular")) { + player->video360_metadata.projection_type = VIDEO360_PROJECTION_TYPE_EQUIRECTANGULAR; + } else { + LOGE("Projection %s: code not implemented.\n", player->video360_metadata.projection_type_string); + player->is_content_spherical = player->is_video360_enabled = FALSE; + } + } + + if (player->video360_metadata.stereo_mode_string) { + if (!strcmp(player->video360_metadata.stereo_mode_string, "mono")) { + player->video360_metadata.stereo_mode = VIDEO360_MODE_MONOSCOPIC; + } else if (!strcmp(player->video360_metadata.stereo_mode_string, "left-right")) { + player->video360_metadata.stereo_mode = VIDEO360_MODE_STEREOSCOPIC_LEFT_RIGHT; + } else if (!strcmp(player->video360_metadata.stereo_mode_string, "top-bottom")) { + player->video360_metadata.stereo_mode = VIDEO360_MODE_STEREOSCOPIC_TOP_BOTTOM; + } else { + LOGE("Stereo mode %s: code not implemented.\n", player->video360_metadata.stereo_mode_string); + player->is_content_spherical = player->is_video360_enabled = FALSE; + } + } + } + } + if (mmf_attrs_commit(attrs)) LOGE("failed to commit.\n"); @@ -4189,6 +4256,9 @@ static void __mmplayer_gst_caps_notify_cb(GstPad * pad, GParamSpec * unused, gpo player->audio_stream_changed_cb(player->audio_stream_changed_cb_user_param); } } else if (strstr(name, "video")) { + if ((name = gst_structure_get_string(str, "format"))) + player->set_mode.video_zc = name[0] == 'S'; + _mmplayer_update_content_attrs(player, ATTR_VIDEO); if (player->video_stream_changed_cb) { @@ -4550,6 +4620,7 @@ __mmplayer_gst_create_audio_pipeline(mm_player_t* player) GList* element_bucket = NULL; gboolean link_audio_sink_now = TRUE; int i = 0; + GstCaps *acaps; MMPLAYER_FENTER(); @@ -4685,7 +4756,7 @@ __mmplayer_gst_create_audio_pipeline(mm_player_t* player) gst_caps_unref(caps); #endif - /* chech if multi-chennels */ + /* check if multi-channels */ if (player->pipeline->mainbin && player->pipeline->mainbin[MMPLAYER_M_DEMUX].gst) { GstPad *srcpad = NULL; GstCaps *caps = NULL; @@ -4724,7 +4795,47 @@ __mmplayer_gst_create_audio_pipeline(mm_player_t* player) } /* create audio sink */ - MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audiosink_element, "audiosink", link_audio_sink_now, player); + + /* Note: qtdemux converts audio metadata defaults to openalsink defaults. */ + if (player->is_content_spherical && + channels == 4 && + player->video360_metadata.ambisonic_type == MMFILE_AMBISONIC_TYPE_PERIPHONIC && + player->video360_metadata.ambisonic_format == MMFILE_AMBISONIC_FORMAT_AMB && + player->video360_metadata.ambisonic_order == MMFILE_AMBISONIC_ORDER_FOA) { + + strncpy(player->ini.audiosink_element, "openalsink", PLAYER_INI_MAX_STRLEN - 1); + + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CONV_BFORMAT, "audioconvert", "audio-converter-bformat", link_audio_sink_now, player); + + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_CAPS_360, "capsfilter", "audio-caps-filter", link_audio_sink_now, player); + acaps = gst_caps_from_string(SPATIAL_AUDIO_CAPS); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_CAPS_360].gst), "caps", acaps, NULL); + gst_caps_unref(acaps); + + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, "openalsink", "audiosink", link_audio_sink_now, player); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "source-ambisonics-type", 1, NULL); + sound_manager_create_stream_information(SOUND_STREAM_TYPE_MEDIA, NULL, NULL, &stream_info); + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "stream-info", stream_info, NULL); + + player->is_openal_plugin_used = TRUE; + + if (player->video360_yaw_radians <= M_PI && + player->video360_yaw_radians >= -M_PI && + player->video360_pitch_radians <= M_PI_2 && + player->video360_pitch_radians >= -M_PI_2) { + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), + "source-orientation-y", (int) (player->video360_yaw_radians * 180.0 / M_PI), + "source-orientation-x", (int) (player->video360_pitch_radians * 180.0 / M_PI), NULL); + } else if (player->video360_metadata.init_view_heading || player->video360_metadata.init_view_pitch) { + g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), + "source-orientation-y", player->video360_metadata.init_view_heading, + "source-orientation-x", player->video360_metadata.init_view_pitch, NULL); + } + } else { + if (player->is_content_spherical) + LOGW("Audio track isn't of the ambisonic type and can't be played back as a spatial sound.\n"); + MMPLAYER_CREATE_ELEMENT(audiobin, MMPLAYER_A_SINK, player->ini.audiosink_element, "audiosink", link_audio_sink_now, player); + } /* qos on */ g_object_set(G_OBJECT(audiobin[MMPLAYER_A_SINK].gst), "qos", TRUE, NULL); /* qos on */ @@ -5145,7 +5256,7 @@ __mmplayer_video_stream_decoded_render_cb(GstElement* object, GstBuffer *buffer, tbm_bo_unmap(stream.bo[0]); } - if (player->video_stream_cb) + if (player->video_stream_cb) /* This has been already checked at the entry */ player->video_stream_cb(&stream, player->video_stream_cb_user_param); if (metaBlock) @@ -5248,6 +5359,61 @@ __mmplayer_gst_create_video_pipeline(mm_player_t* player, GstCaps* caps, MMDispl int enable_video_decoded_cb = 0; mm_attrs_get_int_by_name(player->attrs, "enable_video_decoded_cb", &enable_video_decoded_cb); + if (player->is_content_spherical) { + MMPLAYER_CREATE_ELEMENT(videobin, MMPLAYER_V_360, "video360", + "video-360", TRUE, player); + + player->is_video360_plugin_used = TRUE; + /* Set spatial media metadata and/or user settings to the element. + * */ + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "projection-type", player->video360_metadata.projection_type, NULL); + + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "stereo-mode", player->video360_metadata.stereo_mode, NULL); + + if (player->video360_metadata.full_pano_width_pixels && + player->video360_metadata.full_pano_height_pixels && + player->video360_metadata.cropped_area_image_width && + player->video360_metadata.cropped_area_image_height) { + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "projection-bounds-top", player->video360_metadata.cropped_area_top, + "projection-bounds-bottom", player->video360_metadata.full_pano_height_pixels - + player->video360_metadata.cropped_area_top - player->video360_metadata.cropped_area_image_height, + "projection-bounds-left", player->video360_metadata.cropped_area_left, + "projection-bounds-right", player->video360_metadata.full_pano_width_pixels - + player->video360_metadata.cropped_area_left - player->video360_metadata.cropped_area_image_width, + NULL); + } + + if (player->video360_horizontal_fov && player->video360_vertical_fov) { + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "horizontal-fov", player->video360_horizontal_fov, + "vertical-fov", player->video360_vertical_fov, NULL); + } + + if (player->video360_zoom <= VIDEO360_MAX_ZOOM && player->video360_zoom > 1.0f) { + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "zoom", 1.0f / player->video360_zoom, NULL); + } + + if (player->video360_yaw_radians <= M_PI && + player->video360_yaw_radians >= -M_PI && + player->video360_pitch_radians <= M_PI_2 && + player->video360_pitch_radians >= -M_PI_2) { + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "pose-yaw", (int) (player->video360_yaw_radians * 180.0 / M_PI), + "pose-pitch", (int) (player->video360_pitch_radians * 180.0 / M_PI), NULL); + } else if (player->video360_metadata.init_view_heading || player->video360_metadata.init_view_pitch) { + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "pose-yaw", player->video360_metadata.init_view_heading, + "pose-pitch", player->video360_metadata.init_view_pitch, NULL); + } + + g_object_set(G_OBJECT(videobin[MMPLAYER_V_360].gst), + "passthrough", !player->is_video360_enabled, NULL); + } + /* set video sink */ switch (surface_type) { case MM_DISPLAY_SURFACE_OVERLAY: @@ -6183,19 +6349,22 @@ __mmplayer_gst_create_pipeline(mm_player_t* player) goto INIT_ERROR; } + player->video360_metadata.is_spherical = -1; + player->is_openal_plugin_used = FALSE; + player->is_video360_plugin_used = FALSE; + player->pipeline = (MMPlayerGstPipelineInfo*) g_malloc0(sizeof(MMPlayerGstPipelineInfo)); if (player->pipeline == NULL) goto INIT_ERROR; - memset(player->pipeline, 0, sizeof(MMPlayerGstPipelineInfo)); - + memset(player->pipeline, 0, sizeof(MMPlayerGstPipelineInfo)); /* g_malloc0 did this job already */ /* create mainbin */ mainbin = (MMPlayerGstElement*) g_malloc0(sizeof(MMPlayerGstElement) * MMPLAYER_M_NUM); if (mainbin == NULL) goto INIT_ERROR; - memset(mainbin, 0, sizeof(MMPlayerGstElement) * MMPLAYER_M_NUM); + memset(mainbin, 0, sizeof(MMPlayerGstElement) * MMPLAYER_M_NUM); /* g_malloc0 did this job already */ /* create pipeline */ mainbin[MMPLAYER_M_PIPE].id = MMPLAYER_M_PIPE; @@ -8506,6 +8675,18 @@ _mmplayer_create_player(MMHandleType handle) player->adaptive_info.limit.width = ADAPTIVE_VARIANT_DEFAULT_VALUE; player->adaptive_info.limit.height = ADAPTIVE_VARIANT_DEFAULT_VALUE; + /* Set video360 settings to their defaults for just-created player. + * */ + player->is_content_spherical = FALSE; + player->is_video360_enabled = TRUE; + player->video360_metadata.projection_type = VIDEO360_PROJECTION_TYPE_EQUIRECTANGULAR; + player->video360_metadata.stereo_mode = VIDEO360_MODE_MONOSCOPIC; + player->video360_yaw_radians = 4; + player->video360_pitch_radians = 4; + player->video360_zoom = 1.0f; + player->video360_horizontal_fov = 0; + player->video360_vertical_fov = 0; + /* set player state to null */ MMPLAYER_STATE_CHANGE_TIMEOUT(player) = player->ini.localplayback_state_change_timeout; MMPLAYER_SET_STATE(player, MM_PLAYER_STATE_NULL); @@ -11799,6 +11980,19 @@ __mmplayer_release_misc(mm_player_t* player) player->adaptive_info.limit.width = ADAPTIVE_VARIANT_DEFAULT_VALUE; player->adaptive_info.limit.height = ADAPTIVE_VARIANT_DEFAULT_VALUE; + /* Reset video360 settings to their defaults in case if the pipeline is to be + * re-created. + * */ + player->is_content_spherical = FALSE; + player->is_video360_enabled = TRUE; + player->video360_metadata.projection_type = VIDEO360_PROJECTION_TYPE_EQUIRECTANGULAR; + player->video360_metadata.stereo_mode = VIDEO360_MODE_MONOSCOPIC; + player->video360_yaw_radians = 4; + player->video360_pitch_radians = 4; + player->video360_zoom = 1.0f; + player->video360_horizontal_fov = 0; + player->video360_vertical_fov = 0; + MMPLAYER_FLEAVE(); }