Implement Spherical video metadata V2 extraction in MKV/WebM 01/155001/9
authorMykola Alieksieiev <m.alieksieie@samsung.com>
Fri, 29 Sep 2017 15:18:02 +0000 (18:18 +0300)
committerhj kim <backto.kim@samsung.com>
Tue, 13 Feb 2018 07:52:08 +0000 (07:52 +0000)
- No stereo-mode support added as far as it is stored using the
  existing StereoMode element specified in the Matroska spec. and
  can be supported in the Matroska format parser

- WebM files prepared with MKVMerge tool with "-w" option turned on
  (force WebM sub-format) are not recognized by libmm-fileinfo

- Ambisonic audio is not specified in Google's RFC v2

Change-Id: I4935602afb658bfa6971be247a4ffd1e3332fe55
Signed-off-by: Mykola Alieksieiev <m.alieksieie@samsung.com>
formats/ffmpeg/mm_file_format_ffmpeg.c
utils/include/mm_file_utils.h
utils/mm_file_util_io.c
utils/mm_file_util_tag.c

index ae87e58b1ef82a1e2356b1136fab2ef323ae66ec..a33651730486bf98de2e38516d1b60b84485b940 100755 (executable)
@@ -565,6 +565,10 @@ int mmfile_format_read_tag_ffmpg(MMFileFormatContext *formatContext)
                MMFileUtilGetMetaDataFromMP4(formatContext);
        }
 
+       if (formatContext->formatType == MM_FILE_FORMAT_MATROSKA) {
+               MMFileUtilGetMetaDataFromMKV(formatContext);
+       }
+
        /*metadata extracted by ffmpeg*/
        unsigned int idx = 0;
 
@@ -659,6 +663,8 @@ int mmfile_format_read_tag_ffmpg(MMFileFormatContext *formatContext)
                                                } else if (!strcasecmp(tag->key, "rotate")) {   /*can be "90", "180", "270" */
                                                        if (formatContext->rotate)      free(formatContext->rotate);
                                                        formatContext->rotate = mmfile_strdup(tag->value);
+                                               } else if (!strcasecmp(tag->key, "spherical-video")) {
+                                                       ParseSpatialVideoMetadataFromXMLString(tag->value, formatContext);
                                                } else if (!strcasecmp(tag->key, "metadata_block_picture")) {
                                                        gsize len = 0;
                                                        guchar *meta_data = NULL;
index 73d23342b74987dabc595f8a7a054e22e75081ff..b289a9efac3d45b97f0733ec8b18ba9543bee36d 100755 (executable)
@@ -67,6 +67,7 @@ inline unsigned short   mmfile_io_be_uint16(unsigned short value);
 inline unsigned short   mmfile_io_le_uint16(unsigned short value);
 inline short            mmfile_io_be_int16(unsigned short value);
 inline short            mmfile_io_le_int16(unsigned short value);
+inline float            mmfile_io_be_float32(float value);
 
 typedef struct MMFileIOHandle {
        struct MMFileIOFunc *iofunc;
@@ -456,7 +457,7 @@ typedef struct {
        long long               duration;
 
 /* for mp3 Info */
-       char                    *pToc;                  /* VBR??? SeekPosition?? ????? ???? TOC ??????? ?????? ??\EF\BF???? char ?ò÷, 100 ????? ???? */
+       char                    *pToc;                  /* VBR??? SeekPosition?? ????? ???? TOC ??????? ?????? ??\EF\BF???? char ?��, 100 ????? ???? */
        unsigned int    mpegVersion;    /* 1 : mpeg 1,    2 : mpeg 2, 3 : mpeg2.5 */
        unsigned int    layer;                  /* 1 : layer1, 2 : layer2, 3 : layer3 */
        unsigned int    channelIndex;   /* 0 : stereo, 1 : joint_stereo, 2 : dual_channel, 3 : mono */
@@ -466,7 +467,7 @@ typedef struct {
        long                    headerPos;              /* mp3 ????\EF\BF?o?????? ??????? ??? */
        long long               datafileLen;    /* ID3Tag???? ??????? ???? mp3 frame???? ????,  VBR??? XHEADDATA ?? bytes ?? ?????? */
        int                             frameSize;              /* mp3 frame ?? ???? ??? */
-       int                             frameNum;               /* mp3 ????? ???????? ?? ?????¡Æ\EF\BF? */
+       int                             frameNum;               /* mp3 ????? ???????? ?? ?????��\EF\BF? */
        bool                    bVbr;                   /* VBR mp3? */
        bool                    bPadding;               /* Padding? */
        bool                    bV1tagFound;
@@ -556,7 +557,9 @@ bool        mm_file_id3tag_parse_v222(AvFileContentInfo *pInfo, unsigned char *buffer);
 bool   mm_file_id3tag_parse_v223(AvFileContentInfo *pInfo, unsigned char *buffer);
 bool   mm_file_id3tag_parse_v224(AvFileContentInfo *pInfo, unsigned char *buffer);
 void   mm_file_id3tag_restore_content_info(AvFileContentInfo *pInfo);
+int            MMFileUtilGetMetaDataFromMKV(MMFileFormatContext *formatContext);
 int            MMFileUtilGetMetaDataFromMP4(MMFileFormatContext *formatContext);
+int            ParseSpatialVideoMetadataFromXMLString(const char *xmlStr, MMFileFormatContext *formatContext);
 
 
 #ifdef __cplusplus
index 93f72890f9410522d702325c2c033d8e22985e30..eee55633be712fdc7c1dd0bb6e0ee0f3e047a798 100755 (executable)
@@ -82,8 +82,15 @@ inline short mmfile_io_le_int16(unsigned short value)
        return (is_little_endian == 1) ? value : ((short)((((value) & 0xFF00) >> 8) | (((value) & 0x00FF) << 8)));
 }
 
-
-
+EXPORT_API
+inline float mmfile_io_be_float32(float value)
+{
+       /* Additional variables were introduced to avoid dereferencing type-punned pointer compilation error */
+       float *value_ptr = &value;
+       unsigned int result_uint = ((((*(unsigned int*)value_ptr) & 0x000000FF) << 24) | (((*(unsigned int*)value_ptr) & 0x0000FF00) << 8) | (((*(unsigned int*)value_ptr) & 0x00FF0000) >> 8) | (((*(unsigned int*)value_ptr) & 0xFF000000) >> 24));
+       float *result = (float*)&result_uint;
+       return (is_little_endian == 0) ? value : *result;
+}
 
 MMFileIOFunc *first_io_func = NULL;
 
index 4350586ea2a2faf685c8ba822817a41f0914addf..dd63f2a69f4c6dcabe785afead184f4a677eb018 100755 (executable)
@@ -142,6 +142,43 @@ typedef struct _mmfilesa3dbox {
        uint32_t channel_map[49]; /* Up to 6th order */
 } __attribute__((aligned(1), packed)) MMFILE_MP4A_SA3D_TAGBOX;
 
+typedef struct _mmfile_proj_v2_box {
+       uint16_t proj_box_id;
+       uint8_t proj_box_size;
+       uint16_t proj_type_box_id;
+       uint8_t proj_type_box_size;
+       uint8_t proj_type_box_value; /* only equi (=1) or cubemap (=2) currently */
+       uint16_t proj_priv_box_id;
+       uint8_t proj_priv_box_size;
+} __attribute__((aligned(1), packed)) MMFILE_WEBM_PROJ_V2_BOX;
+
+typedef struct _mmfile_equi_proj_v2_box {
+       unsigned char proj_priv_box_name[4]; /* "equi" */
+       uint32_t equi_projection_bounds_top;
+       uint32_t equi_projection_bounds_bottom;
+       uint32_t equi_projection_bounds_left;
+       uint32_t equi_projection_bounds_right;
+} __attribute__((aligned(1), packed)) MMFILE_WEBM_EQUI_PROJ_V2_BOX;
+
+typedef struct _mmfile_cbmp_proj_v2_box {
+       unsigned char proj_priv_box_name[4]; /* "cbmp" */
+       uint32_t cbmp_projection_layout;
+       uint32_t cbmp_projection_padding;
+} __attribute__((aligned(1), packed)) MMFILE_WEBM_CBMP_PROJ_V2_BOX;
+
+typedef struct _mmfile_pose_element_v2_box {
+       uint16_t pose_yaw_element_id;
+       uint8_t pose_yaw_element_size;
+       float pose_yaw_element_value;
+       uint16_t pose_pitch_element_id;
+       uint8_t pose_pitch_element_size;
+       float  pose_pitch_element_value;
+       uint16_t pose_roll_element_id;
+       uint8_t pose_roll_element_size;
+       float  pose_roll_element_value;
+} __attribute__((aligned(1), packed)) MMFILE_WEBM_POSE_ELEMENT_V2_BOX;
+
+
 typedef enum {
        PROJECTION_TYPE_RECT = 0, /* Rectangular, Google's RFC value */
        PROJECTION_TYPE_EQUI = 1, /* Equirectangular, Google's RFC value */
@@ -150,6 +187,15 @@ typedef enum {
        PROJECTION_TYPE_UNKNOWN = INVALID_UINT_VALUE,
 } SPHERICAL_VIDEO2_PROJECTION_TYPE;
 
+enum spherical_video_metadata_elements_ids_le {
+       PROJ_BOX_ID           = 0x7076,
+       PROJ_TYPE_BOX_ID      = 0x7176,
+       PROJ_PRIV_BOX_ID      = 0x7276,
+       POSE_YAW_ELEMENT_ID   = 0x7376,
+       POSE_PITCH_ELEMENT_ID = 0x7476,
+       POSE_ROLL_ELEMENT_ID  = 0x7576
+};
+
 #define MMFILE_MP4_BASIC_BOX_HEADER_LEN 8
 #define MMFILE_MP4_MOVIE_HEADER_BOX_LEN 96
 #define MMFILE_MP4_HDLR_BOX_LEN         24
@@ -1845,7 +1891,7 @@ static int GetJunkCounterLimit(void)
        return data;
 }
 
-static int ParseSpatialVideoMetadataFromXMLString(const char *xmlStr, MMFileFormatContext *formatContext)
+int ParseSpatialVideoMetadataFromXMLString(const char *xmlStr, MMFileFormatContext *formatContext)
 {
        const char is_spherical_str[] = "<GSpherical:Spherical>";
        const char is_stitched_str[] = "<GSpherical:Stitched>";
@@ -1906,6 +1952,159 @@ static int ParseSpatialVideoMetadataFromXMLString(const char *xmlStr, MMFileForm
 }
 
 #define BIG_CONTENT_BOX_SIZE_LEN 8
+
+int MMFileUtilGetMetaDataFromMKV(MMFileFormatContext *formatContext)
+{
+       MMFileIOHandle *fp = NULL;
+       int probe_size = 10000;
+       unsigned char *buffer = NULL;
+       int ret = 0;
+       int i;
+
+       MMFILE_WEBM_PROJ_V2_BOX v2box = { 0, };
+       MMFILE_WEBM_EQUI_PROJ_V2_BOX equi = { 0, };
+       MMFILE_WEBM_CBMP_PROJ_V2_BOX cbmp = { 0, };
+       MMFILE_WEBM_POSE_ELEMENT_V2_BOX pose = { 0, };
+
+       ret = mmfile_open(&fp, formatContext->uriFileName, MMFILE_RDONLY);
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "error: mmfile_open\n");
+               goto exit;
+       }
+
+       long long file_size = mmfile_seek(fp, 0, SEEK_END);
+       if (file_size == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "mmfile operation failed\n");
+               goto exit;
+       }
+
+       probe_size = (file_size > probe_size) ? probe_size : file_size;
+       buffer = (unsigned char *)malloc(probe_size * sizeof(unsigned char));
+       if (!buffer) {
+               debug_error(DEBUG, "malloc failed\n");
+               goto exit;
+       }
+
+       ret = mmfile_seek(fp, 0, SEEK_SET);
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "mmfile operation failed\n");
+               goto exit;
+       }
+
+       ret = mmfile_read(fp, buffer, probe_size * sizeof(unsigned char));
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "mmfile operation failed\n");
+               goto exit;
+       }
+
+       /* FIXME (m.alieksieie): It's better to use some EBML parser here*/
+       for (i = 0; i + 3 < probe_size; ++i) {
+               if (*(unsigned int *)(buffer + i) == FOURCC('e', 'q', 'u', 'i') ||
+                               *(unsigned int *)(buffer + i) == FOURCC('c', 'b', 'm', 'p')) {
+                       debug_msg(DEBUG, "projection data found at offset %d bytes\n", i);
+                       break;
+               }
+       }
+
+       if (i + 3 == probe_size) {
+               debug_msg(DEBUG, "projection info wasn't found\n");
+               ret = MMFILE_UTIL_SUCCESS;
+               goto exit;
+       }
+
+       if ((i - (int)sizeof(MMFILE_WEBM_PROJ_V2_BOX)) < 0) {
+               debug_error(DEBUG, "error: invalid supposed projection info location\n");
+               ret = MMFILE_UTIL_FAIL;
+               goto exit;
+       }
+
+       ret = mmfile_seek(fp, i - sizeof(MMFILE_WEBM_PROJ_V2_BOX), SEEK_SET);
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "error: failed to seek to the supposed projection info location\n");
+               goto exit;
+       }
+
+       ret = mmfile_read(fp, (unsigned char *)&v2box, sizeof(MMFILE_WEBM_PROJ_V2_BOX));
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "mmfile operation failed\n");
+               goto exit;
+       }
+
+       if (v2box.proj_type_box_value == PROJECTION_TYPE_EQUI) {
+               debug_msg(DEBUG, "Equirectangular projection is used\n");
+
+               ret = mmfile_read(fp, (unsigned char *)&equi, sizeof(MMFILE_WEBM_EQUI_PROJ_V2_BOX));
+               if (ret == MMFILE_UTIL_FAIL) {
+                       debug_error(DEBUG, "error: failed to read equirectangular element\n");
+                       goto exit;
+               }
+               if (strncmp((char *)equi.proj_priv_box_name, "equi", 4) == 0) {
+                       formatContext->equiBoundsTopV2 = mmfile_io_be_uint32(equi.equi_projection_bounds_top);
+                       formatContext->equiBoundsBottomV2 = mmfile_io_be_uint32(equi.equi_projection_bounds_bottom);
+                       formatContext->equiBoundsLeftV2 = mmfile_io_be_uint32(equi.equi_projection_bounds_left);
+                       formatContext->equiBoundsRightV2 = mmfile_io_be_uint32(equi.equi_projection_bounds_right);
+               } else {
+                       debug_error(DEBUG, "error: failed to read equirectangular element\n");
+                       ret = MMFILE_UTIL_SUCCESS;
+                       goto exit;
+               }
+       }
+       if (v2box.proj_type_box_value == PROJECTION_TYPE_CBMP) {
+               debug_msg(DEBUG, "Cubemap projection is used\n");
+
+               ret = mmfile_read(fp, (unsigned char *)&cbmp, sizeof(MMFILE_WEBM_CBMP_PROJ_V2_BOX));
+               if (ret == MMFILE_UTIL_FAIL) {
+                       debug_error(DEBUG, "error: failed to read cubemap element\n");
+                       goto exit;
+               }
+               if (strncmp((char *)cbmp.proj_priv_box_name, "cbmp", 4) == 0) {
+                       formatContext->cbmpLayoutV2 = mmfile_io_be_uint32(cbmp.cbmp_projection_layout);
+                       formatContext->cbmpPaddingV2 = mmfile_io_be_uint32(cbmp.cbmp_projection_padding);
+               } else {
+                       debug_error(DEBUG, "error: failed to read cubemap element\n");
+                       ret = MMFILE_UTIL_FAIL;
+                       goto exit;
+               }
+       }
+
+       ret = mmfile_read(fp, (unsigned char *)&pose, sizeof(MMFILE_WEBM_POSE_ELEMENT_V2_BOX));
+       if (ret == MMFILE_UTIL_FAIL) {
+               debug_error(DEBUG, "error: failed to read pose info\n");
+               goto exit;
+       }
+
+       if (pose.pose_yaw_element_id == POSE_YAW_ELEMENT_ID) {
+               formatContext->poseYawV2 = (uint)mmfile_io_be_float32(pose.pose_yaw_element_value);
+       } else {
+               debug_error(DEBUG, "error: failed to pose yaw element\n");
+               ret = MMFILE_UTIL_FAIL;
+               goto exit;
+       }
+       if (pose.pose_pitch_element_id == POSE_PITCH_ELEMENT_ID) {
+               formatContext->posePitchV2 = (uint)mmfile_io_be_float32(pose.pose_pitch_element_value);
+       } else {
+               debug_error(DEBUG, "error: failed to pose pitch element\n");
+               ret = MMFILE_UTIL_FAIL;
+               goto exit;
+       }
+       if (pose.pose_roll_element_id == POSE_ROLL_ELEMENT_ID) {
+               formatContext->poseRollV2 = (uint)mmfile_io_be_float32(pose.pose_roll_element_value);
+       } else {
+               debug_error(DEBUG, "error: failed to pose roll element\n");
+               ret = MMFILE_UTIL_FAIL;
+               goto exit;
+       }
+
+exit:
+       if (fp)
+               mmfile_close(fp);
+
+       if (buffer)
+               free(buffer);
+
+       return ret;
+}
+
 EXPORT_API int MMFileUtilGetMetaDataFromMP4(MMFileFormatContext *formatContext)
 {
        MMFileIOHandle *fp = NULL;