From d8d3c78a1faf33118268a2bc6a5be4b39c6b704a Mon Sep 17 00:00:00 2001 From: Jeongmo Yang Date: Tue, 28 Jul 2020 10:27:31 +0900 Subject: [PATCH] Add new API for video encode decision callback - API prototype typedef gboolean (*mm_camcorder_video_encode_decision_callback)(MMCamcorderVideoStreamDataType *stream, void *user_param); int mm_camcorder_set_video_encode_decision_callback(MMHandleType camcorder, mm_camcorder_video_encode_decision_callback callback, void* user_data); - Detail of change : Add above APIs : Add sub function to invoke video stream callbacks for preview and video stream [Version] 0.10.208 [Profile] Common [Issue Type] New feature Change-Id: I690dddc872e72cf7ba192a99aa19747673640ac3 Signed-off-by: Jeongmo Yang --- packaging/libmm-camcorder.spec | 2 +- src/include/mm_camcorder.h | 30 ++ src/include/mm_camcorder_gstcommon.h | 1 + src/include/mm_camcorder_internal.h | 21 ++ src/include/mm_camcorder_videorec.h | 1 + src/mm_camcorder.c | 8 + src/mm_camcorder_gstcommon.c | 542 +++++++++++++-------------- src/mm_camcorder_internal.c | 28 ++ src/mm_camcorder_videorec.c | 31 +- 9 files changed, 364 insertions(+), 300 deletions(-) diff --git a/packaging/libmm-camcorder.spec b/packaging/libmm-camcorder.spec index 7219da9..6330607 100644 --- a/packaging/libmm-camcorder.spec +++ b/packaging/libmm-camcorder.spec @@ -1,6 +1,6 @@ Name: libmm-camcorder Summary: Camera and recorder library -Version: 0.10.207 +Version: 0.10.208 Release: 0 Group: Multimedia/Libraries License: Apache-2.0 diff --git a/src/include/mm_camcorder.h b/src/include/mm_camcorder.h index ea51fdc..0d9dda1 100644 --- a/src/include/mm_camcorder.h +++ b/src/include/mm_camcorder.h @@ -1941,6 +1941,17 @@ typedef gboolean (*mm_camcorder_muxed_stream_callback)(MMCamcorderMuxedStreamDat */ typedef gboolean (*mm_camcorder_video_capture_callback)(MMCamcorderCaptureDataType *frame, MMCamcorderCaptureDataType *thumbnail, void *user_param); +/** + * Function definition for video encode decision callback. + * Like '#mm_camcorder_video_stream_callback', you can't call mm_camcorder_stop() while you are hanging this function. + * + * @param[in] stream Reference pointer to video stream data + * @param[in] user_param User parameter which is received from user when callback function was set + * @return This function returns true on encoding, or false on drop frame. + * @remarks This function is issued in the context of gstreamer (video sink or internal of camerasrc thread). + */ +typedef gboolean (*mm_camcorder_video_encode_decision_callback)(MMCamcorderVideoStreamDataType *stream, void *user_param); + /*======================================================================================= | GLOBAL FUNCTION PROTOTYPES | @@ -2988,6 +2999,25 @@ gboolean setting_muxed_stream_callback() int mm_camcorder_set_muxed_stream_callback(MMHandleType camcorder, mm_camcorder_muxed_stream_callback callback, void *user_data); +/** + * mm_camcorder_set_video_encode_decision_callback:\n + * Set callback for user defined video encode decision callback function. + * Users can retrieve video frame using registered callback, + * and decide to encoding video frame by return value of function. + * + * @param[in] camcorder A handle of camcorder. + * @param[in] callback Function pointer of callback function. + * @param[in] user_data User parameter for passing to callback function. + * @return This function returns zero(MM_ERROR_NONE) on success, or negative value with error code.\n + * Please refer 'mm_error.h' to know the exact meaning of the error. + * @see mm_camcorder_video_encode_decision_callback + * @pre None + * @post None + * @remarks registered 'callback' is called on internal thread of camcorder. Regardless of the status of main loop, this function will be called. + */ +int mm_camcorder_set_video_encode_decision_callback(MMHandleType camcorder, mm_camcorder_video_encode_decision_callback callback, void *user_data); + + /** * mm_camcorder_get_state:\n * Get the current state of camcorder. diff --git a/src/include/mm_camcorder_gstcommon.h b/src/include/mm_camcorder_gstcommon.h index 7babef8..4cd71bf 100644 --- a/src/include/mm_camcorder_gstcommon.h +++ b/src/include/mm_camcorder_gstcommon.h @@ -181,6 +181,7 @@ bool _mmcamcorder_set_encoded_preview_bitrate(MMHandleType handle, int bitrate); bool _mmcamcorder_set_encoded_preview_gop_interval(MMHandleType handle, int gop_interval); bool _mmcamcorder_set_sound_stream_info(GstElement *element, char *stream_type, int stream_index); void _mmcamcorder_set_encoder_bitrate(MMCamcorderEncoderType type, int codec, int bitrate, GstElement *element); +gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstBuffer *buffer, gboolean is_preview); GstPadProbeReturn __mmcamcorder_muxed_dataprobe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data); GstPadProbeReturn __mmcamcorder_eventprobe_monitor(GstPad *pad, GstPadProbeInfo *info, gpointer u_data); #ifdef __cplusplus diff --git a/src/include/mm_camcorder_internal.h b/src/include/mm_camcorder_internal.h index b594a83..7edc812 100644 --- a/src/include/mm_camcorder_internal.h +++ b/src/include/mm_camcorder_internal.h @@ -444,6 +444,11 @@ do { \ #define _MMCAMCORDER_TRYLOCK_MSTREAM_CALLBACK(handle) _MMCAMCORDER_TRYLOCK_FUNC(_MMCAMCORDER_GET_MSTREAM_CALLBACK_LOCK(handle)) #define _MMCAMCORDER_UNLOCK_MSTREAM_CALLBACK(handle) _MMCAMCORDER_UNLOCK_FUNC(_MMCAMCORDER_GET_MSTREAM_CALLBACK_LOCK(handle)) +#define _MMCAMCORDER_GET_VEDECISION_CALLBACK_LOCK(handle) (_MMCAMCORDER_CAST_MTSAFE(handle).vedecision_cb_lock) +#define _MMCAMCORDER_LOCK_VEDECISION_CALLBACK(handle) _MMCAMCORDER_LOCK_FUNC(_MMCAMCORDER_GET_VEDECISION_CALLBACK_LOCK(handle)) +#define _MMCAMCORDER_TRYLOCK_VEDECISION_CALLBACK(handle) _MMCAMCORDER_TRYLOCK_FUNC(_MMCAMCORDER_GET_VEDECISION_CALLBACK_LOCK(handle)) +#define _MMCAMCORDER_UNLOCK_VEDECISION_CALLBACK(handle) _MMCAMCORDER_UNLOCK_FUNC(_MMCAMCORDER_GET_VEDECISION_CALLBACK_LOCK(handle)) + #ifdef _MMCAMCORDER_MM_RM_SUPPORT /* for resource conflict */ #define _MMCAMCORDER_GET_RESOURCE_LOCK(handle) (_MMCAMCORDER_CAST_MTSAFE(handle).resource_lock) @@ -657,6 +662,7 @@ typedef struct { GMutex vstream_cb_lock; /**< Mutex (for video stream callback) */ GMutex astream_cb_lock; /**< Mutex (for audio stream callback) */ GMutex mstream_cb_lock; /**< Mutex (for muxed stream callback) */ + GMutex vedecision_cb_lock; /**< Mutex (for video encode decision callback) */ #ifdef _MMCAMCORDER_MM_RM_SUPPORT GMutex resource_lock; /**< Mutex (for resource check) */ #endif /* _MMCAMCORDER_MM_RM_SUPPORT */ @@ -760,6 +766,8 @@ typedef struct mmf_camcorder { void *mstream_cb_param; /**< Muxed stream callback parameter */ mm_camcorder_video_capture_callback vcapture_cb; /**< Video capture callback */ void *vcapture_cb_param; /**< Video capture callback parameter */ + mm_camcorder_video_encode_decision_callback vedecision_cb; /**< Video encode decision callback */ + void *vedecision_cb_param; /**< Video encode decision callback parameter */ int (*command)(MMHandleType, int); /**< camcorder's command */ /* etc */ @@ -1097,6 +1105,19 @@ int _mmcamcorder_set_video_capture_callback(MMHandleType hcamcorder, mm_camcorder_video_capture_callback callback, void *user_data); +/** + * This function is to set callback for video encode decision. + * + * @param[in] hcamcorder Specifies the camcorder handle + * @param[in] callback Specifies the function pointer of callback function + * @param[in] user_data Specifies the user poiner for passing to callback function + * + * @return This function returns zero on success, or negative value with error code. + */ +int _mmcamcorder_set_video_encode_decision_callback(MMHandleType hcamcorder, + mm_camcorder_video_encode_decision_callback callback, + void *user_data); + /** * This function returns current state of camcorder, or negative value with error code. * diff --git a/src/include/mm_camcorder_videorec.h b/src/include/mm_camcorder_videorec.h index 630fa16..c148c64 100644 --- a/src/include/mm_camcorder_videorec.h +++ b/src/include/mm_camcorder_videorec.h @@ -61,6 +61,7 @@ typedef struct { double record_motion_rate; /**< motion rate of video recording for slow/fast motion recording */ GstClockTime prev_preview_ts; /**< previous preview frame timestamp */ GstClockTime base_video_ts; /**< base video frame timestamp */ + GstClockTime last_video_ts; /**< last video timestamp which is not modified */ guint record_drop_count; /**< drop count of video recording for slow/fast motion recording */ guint64 video_frame_count; /**< current video frame */ guint64 audio_frame_count; /**< current audio frame */ diff --git a/src/mm_camcorder.c b/src/mm_camcorder.c index 057bf4c..2a68051 100644 --- a/src/mm_camcorder.c +++ b/src/mm_camcorder.c @@ -334,6 +334,14 @@ int mm_camcorder_set_video_capture_callback(MMHandleType camcorder, mm_camcorder } +int mm_camcorder_set_video_encode_decision_callback(MMHandleType camcorder, mm_camcorder_video_encode_decision_callback callback, void* user_data) +{ + mmf_return_val_if_fail((void *)camcorder, MM_ERROR_CAMCORDER_INVALID_ARGUMENT); + + return _mmcamcorder_set_video_encode_decision_callback(camcorder, callback, user_data); +} + + int mm_camcorder_get_state(MMHandleType camcorder, MMCamcorderStateType *state) { int ret = MM_ERROR_NONE; diff --git a/src/mm_camcorder_gstcommon.c b/src/mm_camcorder_gstcommon.c index f324b2b..9dc9b47 100644 --- a/src/mm_camcorder_gstcommon.c +++ b/src/mm_camcorder_gstcommon.c @@ -20,7 +20,7 @@ */ /*======================================================================================= -| INCLUDE FILES | +| INCLUDE FILES | =======================================================================================*/ #include #include @@ -37,7 +37,7 @@ #include "mm_camcorder_gstcommon.h" /*----------------------------------------------------------------------- -| GLOBAL VARIABLE DEFINITIONS for internal | +| GLOBAL VARIABLE DEFINITIONS for internal | -----------------------------------------------------------------------*/ /* Table for compatibility between audio codec and file format */ gboolean audiocodec_fileformat_compatibility_table[MM_AUDIO_CODEC_NUM][MM_FILE_FORMAT_NUM] = { @@ -123,7 +123,7 @@ gboolean videocodec_fileformat_compatibility_table[MM_VIDEO_CODEC_NUM][MM_FILE_F /*----------------------------------------------------------------------- -| LOCAL VARIABLE DEFINITIONS for internal | +| LOCAL VARIABLE DEFINITIONS for internal | -----------------------------------------------------------------------*/ #define USE_AUDIO_CLOCK_TUNE #define _MMCAMCORDER_WAIT_EOS_TIME 60.0 /* sec */ @@ -133,7 +133,7 @@ gboolean videocodec_fileformat_compatibility_table[MM_VIDEO_CODEC_NUM][MM_FILE_F /*----------------------------------------------------------------------- -| LOCAL FUNCTION PROTOTYPES: | +| LOCAL FUNCTION PROTOTYPES: | -----------------------------------------------------------------------*/ /* STATIC INTERNAL FUNCTION */ /** @@ -157,12 +157,247 @@ static guint32 _mmcamcorder_convert_fourcc_string_to_value(const gchar* format_n static bool __mmcamcorder_find_max_resolution(MMHandleType handle, gint *max_width, gint *max_height); #endif /* _MMCAMCORDER_PRODUCT_TV */ +static gboolean __mmcamcorder_set_stream_data_tbm(MMCamcorderVideoStreamDataType *stream, tbm_surface_info_s *ts_info); +static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMapInfo *map_info); + /*======================================================================================= -| FUNCTION DEFINITIONS | +| FUNCTION DEFINITIONS | =======================================================================================*/ -/*----------------------------------------------------------------------- -| GLOBAL FUNCTION DEFINITIONS: | ------------------------------------------------------------------------*/ +static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMapInfo *map_info) +{ + mmf_return_val_if_fail(buffer, FALSE); + mmf_return_val_if_fail(map_info, FALSE); + mmf_return_val_if_fail(stream, FALSE); + + switch (stream->format) { + case MM_PIXEL_FORMAT_NV12: /* fall through */ + case MM_PIXEL_FORMAT_NV21: + stream->data.yuv420sp.y = map_info->data; + stream->data.yuv420sp.length_y = stream->width * stream->height; + stream->data.yuv420sp.uv = stream->data.yuv420sp.y + stream->data.yuv420sp.length_y; + stream->data.yuv420sp.length_uv = stream->data.yuv420sp.length_y >> 1; + stream->stride[0] = stream->width; + stream->elevation[0] = stream->height; + stream->stride[1] = stream->width; + stream->elevation[1] = stream->height >> 1; + stream->num_planes = 2; + break; + + case MM_PIXEL_FORMAT_I420: + stream->data.yuv420p.y = map_info->data; + stream->data.yuv420p.length_y = stream->width * stream->height; + stream->data.yuv420p.u = stream->data.yuv420p.y + stream->data.yuv420p.length_y; + stream->data.yuv420p.length_u = stream->data.yuv420p.length_y >> 2; + stream->data.yuv420p.v = stream->data.yuv420p.u + stream->data.yuv420p.length_u; + stream->data.yuv420p.length_v = stream->data.yuv420p.length_u; + stream->stride[0] = stream->width; + stream->elevation[0] = stream->height; + stream->stride[1] = stream->stride[2] = stream->width >> 1; + stream->elevation[1] = stream->elevation[2] = stream->height >> 1; + stream->num_planes = 3; + break; + + case MM_PIXEL_FORMAT_YUYV: /* fall through */ + case MM_PIXEL_FORMAT_UYVY: /* fall through */ + case MM_PIXEL_FORMAT_422P: /* fall through */ + case MM_PIXEL_FORMAT_ITLV_JPEG_UYVY: + stream->data_type = MM_CAM_STREAM_DATA_YUV422; + stream->data.yuv422.yuv = map_info->data; + stream->data.yuv422.length_yuv = stream->length_total; + stream->stride[0] = stream->width << 1; + stream->elevation[0] = stream->height; + stream->num_planes = 1; + break; + + case MM_PIXEL_FORMAT_ENCODED_H264: /* fall through */ + case MM_PIXEL_FORMAT_ENCODED_MJPEG: + stream->data_type = MM_CAM_STREAM_DATA_ENCODED; + stream->data.encoded.data = map_info->data; + stream->data.encoded.length_data = stream->length_total; + stream->data.encoded.is_delta_frame = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); + stream->num_planes = 1; + break; + + case MM_PIXEL_FORMAT_INVZ: + stream->data_type = MM_CAM_STREAM_DATA_DEPTH; + stream->data.depth.data = map_info->data; + stream->data.depth.length_data = stream->length_total; + stream->stride[0] = stream->width << 1; + stream->elevation[0] = stream->height; + stream->num_planes = 1; + break; + + case MM_PIXEL_FORMAT_RGBA: /* fall through */ + case MM_PIXEL_FORMAT_ARGB: + stream->data_type = MM_CAM_STREAM_DATA_RGB; + stream->data.rgb.data = map_info->data; + stream->data.rgb.length_data = stream->length_total; + stream->stride[0] = stream->width << 2; + stream->elevation[0] = stream->height; + stream->num_planes = 1; + break; + + default: + _mmcam_dbg_err("unsupported format[%d]", stream->format); + return FALSE; + } + + return TRUE; +} + + +static gboolean __mmcamcorder_set_stream_data_tbm(MMCamcorderVideoStreamDataType *stream, tbm_surface_info_s *ts_info) +{ + mmf_return_val_if_fail(ts_info, FALSE); + mmf_return_val_if_fail(stream, FALSE); + + switch (stream->format) { + case MM_PIXEL_FORMAT_NV12: /* fall through */ + case MM_PIXEL_FORMAT_NV21: + stream->data_type = MM_CAM_STREAM_DATA_YUV420SP; + stream->num_planes = 2; + stream->data.yuv420sp.y = ts_info->planes[0].ptr; + stream->data.yuv420sp.length_y = ts_info->planes[0].size; + stream->data.yuv420sp.uv = ts_info->planes[1].ptr; + stream->data.yuv420sp.length_uv = ts_info->planes[1].size; + break; + + case MM_PIXEL_FORMAT_I420: + stream->data_type = MM_CAM_STREAM_DATA_YUV420P; + stream->num_planes = 3; + stream->data.yuv420p.y = ts_info->planes[0].ptr; + stream->data.yuv420p.length_y = ts_info->planes[0].size; + stream->data.yuv420p.u = ts_info->planes[1].ptr; + stream->data.yuv420p.length_u = ts_info->planes[1].size; + stream->data.yuv420p.v = ts_info->planes[2].ptr; + stream->data.yuv420p.length_v = ts_info->planes[2].size; + break; + + default: + _mmcam_dbg_err("unsupported format[%d]", stream->format); + return FALSE; + } + + return TRUE; +} + + +gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstBuffer *buffer, gboolean is_preview) +{ + int i = 0; + int num_bos = 0; + gboolean ret_cb = TRUE; + mmf_camcorder_t *hcamcorder = MMF_CAMCORDER(handle); + _MMCamcorderSubContext *sc = NULL; + MMCamcorderVideoStreamDataType stream; + + tbm_surface_h t_surface = NULL; + tbm_surface_info_s ts_info; + + GstMemory *memory = NULL; + GstMapInfo map_info; + + mmf_return_val_if_fail(hcamcorder, FALSE); + mmf_return_val_if_fail(buffer, FALSE); + mmf_return_val_if_fail(gst_buffer_n_memory(buffer), FALSE); + + sc = MMF_CAMCORDER_SUBCONTEXT(hcamcorder); + mmf_return_val_if_fail(sc, FALSE); + + /* clear data structure */ + memset(&map_info, 0x0, sizeof(GstMapInfo)); + memset(&stream, 0x0, sizeof(MMCamcorderVideoStreamDataType)); + + stream.format = sc->info_image->preview_format; + if (is_preview) { + /* preview buffer */ + stream.width = sc->info_video->preview_width; + stream.height = sc->info_video->preview_height; + } else { + /* video recording buffer */ + stream.width = sc->info_video->video_width; + stream.height = sc->info_video->video_height; + } + + /* + _mmcam_dbg_log("VideoStreamData : resolution[%dx%d], format[%d]", + stream.width, stream.height, stream.format); + */ + + /* set size and timestamp */ + if (_mmcamcorder_is_encoded_preview_pixel_format(stream.format)) + memory = gst_buffer_get_all_memory(buffer); + else + memory = gst_buffer_peek_memory(buffer, 0); + if (!memory) { + _mmcam_dbg_err("GstMemory get failed from buffer %p", buffer); + return FALSE; + } + + /* set zero-copy related information */ + if (hcamcorder->use_zero_copy_format) { + t_surface = (tbm_surface_h)gst_tizen_memory_get_surface(memory); + + if (tbm_surface_get_info(t_surface, &ts_info) != TBM_SURFACE_ERROR_NONE) { + _mmcam_dbg_err("failed to get tbm surface[%p] info", t_surface); + goto _INVOKE_VIDEO_STREAM_CB_DONE; + } + + /* set bo, stride and elevation */ + num_bos = gst_tizen_memory_get_num_bos(memory); + for (i = 0 ; i < num_bos ; i++) + stream.bo[i] = gst_tizen_memory_get_bos(memory, i); + + for (i = 0 ; i < ts_info.num_planes ; i++) { + stream.stride[i] = ts_info.planes[i].stride; + stream.elevation[i] = ts_info.planes[i].size / ts_info.planes[i].stride; + /*_mmcam_dbg_log("[%d] %dx%d", i, stream.stride[i], stream.elevation[i]);*/ + } + + stream.length_total = ts_info.size; + stream.internal_buffer = buffer; + + if (!__mmcamcorder_set_stream_data_tbm(&stream, &ts_info)) + goto _INVOKE_VIDEO_STREAM_CB_DONE; + } else { + stream.length_total = gst_memory_get_sizes(memory, NULL, NULL); + + if (!gst_memory_map(memory, &map_info, GST_MAP_READWRITE) || + !__mmcamcorder_set_stream_data_normal(&stream, buffer, &map_info)) + goto _INVOKE_VIDEO_STREAM_CB_DONE; + } + + stream.timestamp = (unsigned int)(GST_BUFFER_PTS(buffer) / 1000000); /* nano sec -> milli sec */ + + /* invoke application callback */ + if (is_preview) { + _MMCAMCORDER_LOCK_VSTREAM_CALLBACK(hcamcorder); + if (hcamcorder->vstream_cb) + hcamcorder->vstream_cb(&stream, hcamcorder->vstream_cb_param); + _MMCAMCORDER_UNLOCK_VSTREAM_CALLBACK(hcamcorder); + } else { + _MMCAMCORDER_LOCK_VEDECISION_CALLBACK(hcamcorder); + if (hcamcorder->vedecision_cb) + ret_cb = hcamcorder->vedecision_cb(&stream, hcamcorder->vedecision_cb_param); + _MMCAMCORDER_UNLOCK_VEDECISION_CALLBACK(hcamcorder); + } + +_INVOKE_VIDEO_STREAM_CB_DONE: + for (i = 0 ; i < TBM_SURF_PLANE_MAX && stream.bo[i] ; i++) { + tbm_bo_map(stream.bo[i], TBM_DEVICE_CPU, TBM_OPTION_READ|TBM_OPTION_WRITE); + tbm_bo_unmap(stream.bo[i]); + } + + if (map_info.data) + gst_memory_unmap(memory, &map_info); + + if (_mmcamcorder_is_encoded_preview_pixel_format(stream.format)) + gst_memory_unref(memory); + + return ret_cb; +} + + int _mmcamcorder_create_preview_elements(MMHandleType handle) { int err = MM_ERROR_NONE; @@ -1676,9 +1911,6 @@ static guint32 _mmcamcorder_convert_fourcc_string_to_value(const gchar* format_n static GstPadProbeReturn __mmcamcorder_video_dataprobe_preview(GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { - int current_state = MM_CAMCORDER_STATE_NONE; - int i = 0; - mmf_camcorder_t *hcamcorder = MMF_CAMCORDER(u_data); _MMCamcorderSubContext *sc = NULL; _MMCamcorderKPIMeasure *kpi = NULL; @@ -1691,8 +1923,6 @@ static GstPadProbeReturn __mmcamcorder_video_dataprobe_preview(GstPad *pad, GstP sc = MMF_CAMCORDER_SUBCONTEXT(u_data); mmf_return_val_if_fail(sc, GST_PAD_PROBE_DROP); - current_state = hcamcorder->state; - if (sc->drop_vframe > 0) { if (sc->pass_first_vframe > 0) { sc->pass_first_vframe--; @@ -1708,7 +1938,7 @@ static GstPadProbeReturn __mmcamcorder_video_dataprobe_preview(GstPad *pad, GstP return GST_PAD_PROBE_DROP; } - if (current_state >= MM_CAMCORDER_STATE_PREPARE) { + if (hcamcorder->state >= MM_CAMCORDER_STATE_PREPARE) { int diff_sec; int frame_count = 0; struct timeval current_video_time; @@ -1744,283 +1974,17 @@ static GstPadProbeReturn __mmcamcorder_video_dataprobe_preview(GstPad *pad, GstP } } - /* video stream callback */ - if (hcamcorder->vstream_cb && buffer) { - int state = MM_CAMCORDER_STATE_NULL; - int num_bos = 0; - unsigned int fourcc = 0; - const gchar *string_format = NULL; - - MMCamcorderVideoStreamDataType stream; - tbm_surface_h t_surface = NULL; - tbm_surface_info_s t_info; - - GstCaps *caps = NULL; - GstStructure *structure = NULL; - GstMemory *memory = NULL; - GstMapInfo mapinfo; - - if (!_mmcamcorder_is_encoded_preview_pixel_format(sc->info_image->preview_format)) { - state = _mmcamcorder_get_state((MMHandleType)hcamcorder); - if (state < MM_CAMCORDER_STATE_PREPARE) { - _mmcam_dbg_warn("Not ready for stream callback"); - return GST_PAD_PROBE_OK; - } - } - - caps = gst_pad_get_current_caps(pad); - if (caps == NULL) { - _mmcam_dbg_warn("Caps is NULL."); - return GST_PAD_PROBE_OK; - } - - /* clear data structure */ - memset(&mapinfo, 0x0, sizeof(GstMapInfo)); - memset(&stream, 0x0, sizeof(MMCamcorderVideoStreamDataType)); - - structure = gst_caps_get_structure(caps, 0); - gst_structure_get_int(structure, "width", &(stream.width)); - gst_structure_get_int(structure, "height", &(stream.height)); - - if (sc->info_image->preview_format == MM_PIXEL_FORMAT_ENCODED_H264) { - stream.format = MM_PIXEL_FORMAT_ENCODED_H264; - } else if (sc->info_image->preview_format == MM_PIXEL_FORMAT_ENCODED_MJPEG) { - stream.format = MM_PIXEL_FORMAT_ENCODED_MJPEG; - } else { - string_format = gst_structure_get_string(structure, "format"); - if (string_format == NULL) { - gst_caps_unref(caps); - caps = NULL; - _mmcam_dbg_warn("get string error!!"); - return GST_PAD_PROBE_OK; - } - - fourcc = _mmcamcorder_convert_fourcc_string_to_value(string_format); - stream.format = _mmcamcorder_get_pixtype(fourcc); - } - - gst_caps_unref(caps); - caps = NULL; - - /* - _mmcam_dbg_log("Call video steramCb, data[%p], Width[%d],Height[%d], Format[%d]", - GST_BUFFER_DATA(buffer), stream.width, stream.height, stream.format); - */ - - if (stream.width == 0 || stream.height == 0) { - _mmcam_dbg_warn("Wrong condition!!"); - return GST_PAD_PROBE_OK; - } - - /* set size and timestamp */ - if (_mmcamcorder_is_encoded_preview_pixel_format(sc->info_image->preview_format)) - memory = gst_buffer_get_all_memory(buffer); - else - memory = gst_buffer_peek_memory(buffer, 0); - if (!memory) { - _mmcam_dbg_err("GstMemory get failed from buffer %p", buffer); - return GST_PAD_PROBE_OK; - } - - if (hcamcorder->use_zero_copy_format) { - t_surface = (tbm_surface_h)gst_tizen_memory_get_surface(memory); - - if (tbm_surface_get_info(t_surface, &t_info) != TBM_SURFACE_ERROR_NONE) { - _mmcam_dbg_err("failed to get tbm surface[%p] info", t_surface); - return GST_PAD_PROBE_OK; - } - - stream.length_total = t_info.size; - - /* set bo, stride and elevation */ - num_bos = gst_tizen_memory_get_num_bos(memory); - for (i = 0 ; i < num_bos ; i++) - stream.bo[i] = gst_tizen_memory_get_bos(memory, i); - - for (i = 0 ; i < t_info.num_planes ; i++) { - stream.stride[i] = t_info.planes[i].stride; - stream.elevation[i] = t_info.planes[i].size / t_info.planes[i].stride; - /*_mmcam_dbg_log("[%d] %dx%d", i, stream.stride[i], stream.elevation[i]);*/ - } - - /* set gst buffer */ - stream.internal_buffer = buffer; - } else { - stream.length_total = gst_memory_get_sizes(memory, NULL, NULL); - } - - stream.timestamp = (unsigned int)(GST_BUFFER_PTS(buffer)/1000000); /* nano sec -> mili sec */ - - /* set data pointers */ - if (stream.format == MM_PIXEL_FORMAT_NV12 || - stream.format == MM_PIXEL_FORMAT_NV21 || - stream.format == MM_PIXEL_FORMAT_I420) { - if (hcamcorder->use_zero_copy_format) { - if (stream.format == MM_PIXEL_FORMAT_NV12 || - stream.format == MM_PIXEL_FORMAT_NV21) { - stream.data_type = MM_CAM_STREAM_DATA_YUV420SP; - stream.num_planes = 2; - stream.data.yuv420sp.y = t_info.planes[0].ptr; - stream.data.yuv420sp.length_y = t_info.planes[0].size; - stream.data.yuv420sp.uv = t_info.planes[1].ptr; - stream.data.yuv420sp.length_uv = t_info.planes[1].size; - /* - _mmcam_dbg_log("format[%d][num_planes:%d] [Y]p:%p,size:%d [UV]p:%p,size:%d", - stream.format, stream.num_planes, - stream.data.yuv420sp.y, stream.data.yuv420sp.length_y, - stream.data.yuv420sp.uv, stream.data.yuv420sp.length_uv); - */ - } else { - stream.data_type = MM_CAM_STREAM_DATA_YUV420P; - stream.num_planes = 3; - stream.data.yuv420p.y = t_info.planes[0].ptr; - stream.data.yuv420p.length_y = t_info.planes[0].size; - stream.data.yuv420p.u = t_info.planes[1].ptr; - stream.data.yuv420p.length_u = t_info.planes[1].size; - stream.data.yuv420p.v = t_info.planes[2].ptr; - stream.data.yuv420p.length_v = t_info.planes[2].size; - /* - _mmcam_dbg_log("S420[num_planes:%d] [Y]p:%p,size:%d [U]p:%p,size:%d [V]p:%p,size:%d", - stream.num_planes, - stream.data.yuv420p.y, stream.data.yuv420p.length_y, - stream.data.yuv420p.u, stream.data.yuv420p.length_u, - stream.data.yuv420p.v, stream.data.yuv420p.length_v); - */ - } - } else { - gst_memory_map(memory, &mapinfo, GST_MAP_READWRITE); - if (stream.format == MM_PIXEL_FORMAT_NV12 || - stream.format == MM_PIXEL_FORMAT_NV21) { - stream.data_type = MM_CAM_STREAM_DATA_YUV420SP; - stream.num_planes = 2; - stream.data.yuv420sp.y = mapinfo.data; - stream.data.yuv420sp.length_y = stream.width * stream.height; - stream.data.yuv420sp.uv = stream.data.yuv420sp.y + stream.data.yuv420sp.length_y; - stream.data.yuv420sp.length_uv = stream.data.yuv420sp.length_y >> 1; - stream.stride[0] = stream.width; - stream.elevation[0] = stream.height; - stream.stride[1] = stream.width; - stream.elevation[1] = stream.height >> 1; - /* - _mmcam_dbg_log("format[%d][num_planes:%d] [Y]p:%p,size:%d [UV]p:%p,size:%d", - stream.format, stream.num_planes, - stream.data.yuv420sp.y, stream.data.yuv420sp.length_y, - stream.data.yuv420sp.uv, stream.data.yuv420sp.length_uv); - */ - } else { - stream.data_type = MM_CAM_STREAM_DATA_YUV420P; - stream.num_planes = 3; - stream.data.yuv420p.y = mapinfo.data; - stream.data.yuv420p.length_y = stream.width * stream.height; - stream.data.yuv420p.u = stream.data.yuv420p.y + stream.data.yuv420p.length_y; - stream.data.yuv420p.length_u = stream.data.yuv420p.length_y >> 2; - stream.data.yuv420p.v = stream.data.yuv420p.u + stream.data.yuv420p.length_u; - stream.data.yuv420p.length_v = stream.data.yuv420p.length_u; - stream.stride[0] = stream.width; - stream.elevation[0] = stream.height; - stream.stride[1] = stream.width >> 1; - stream.elevation[1] = stream.height >> 1; - stream.stride[2] = stream.width >> 1; - stream.elevation[2] = stream.height >> 1; - /* - _mmcam_dbg_log("I420[num_planes:%d] [Y]p:%p,size:%d [U]p:%p,size:%d [V]p:%p,size:%d", - stream.num_planes, - stream.data.yuv420p.y, stream.data.yuv420p.length_y, - stream.data.yuv420p.u, stream.data.yuv420p.length_u, - stream.data.yuv420p.v, stream.data.yuv420p.length_v); - */ - } - } - } else { - gst_memory_map(memory, &mapinfo, GST_MAP_READWRITE); - - switch (stream.format) { - case MM_PIXEL_FORMAT_YUYV: - case MM_PIXEL_FORMAT_UYVY: - case MM_PIXEL_FORMAT_422P: - case MM_PIXEL_FORMAT_ITLV_JPEG_UYVY: - stream.data_type = MM_CAM_STREAM_DATA_YUV422; - stream.data.yuv422.yuv = mapinfo.data; - stream.data.yuv422.length_yuv = stream.length_total; - stream.stride[0] = stream.width << 1; - stream.elevation[0] = stream.height; - break; - case MM_PIXEL_FORMAT_ENCODED_H264: - stream.data_type = MM_CAM_STREAM_DATA_ENCODED; - stream.data.encoded.data = mapinfo.data; - stream.data.encoded.length_data = stream.length_total; - stream.data.encoded.is_delta_frame = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); - /* - _mmcam_dbg_log("H264[num_planes:%d] [0]p:%p,size:%d,is_delta:%d", - stream.num_planes, - stream.data.encoded.data, - stream.data.encoded.length_data, - stream.data.encoded.is_delta_frame); - - */ - break; - case MM_PIXEL_FORMAT_ENCODED_MJPEG: - stream.data_type = MM_CAM_STREAM_DATA_ENCODED; - stream.data.encoded.data = mapinfo.data; - stream.data.encoded.length_data = stream.length_total; - /* - _mmcam_dbg_log("MJPEG[num_planes:%d] [0]p:%p,size:%d", - stream.num_planes, stream.data.encoded.data, stream.data.encoded.length_data); - */ - break; - case MM_PIXEL_FORMAT_INVZ: - stream.data_type = MM_CAM_STREAM_DATA_DEPTH; - stream.data.depth.data = mapinfo.data; - stream.data.depth.length_data = stream.length_total; - stream.stride[0] = stream.width << 1; - stream.elevation[0] = stream.height; - break; - case MM_PIXEL_FORMAT_RGBA: - case MM_PIXEL_FORMAT_ARGB: - stream.data_type = MM_CAM_STREAM_DATA_RGB; - stream.data.rgb.data = mapinfo.data; - stream.data.rgb.length_data = stream.length_total; - stream.stride[0] = stream.width << 2; - stream.elevation[0] = stream.height; - break; - default: - stream.data_type = MM_CAM_STREAM_DATA_YUV420; - stream.data.yuv420.yuv = mapinfo.data; - stream.data.yuv420.length_yuv = stream.length_total; - stream.stride[0] = (stream.width * 3) >> 1; - stream.elevation[0] = stream.height; - break; - } - - stream.num_planes = 1; - /* - _mmcam_dbg_log("%c%c%c%c[num_planes:%d] [0]p:%p,size:%d", - fourcc, fourcc>>8, fourcc>>16, fourcc>>24, - stream.num_planes, stream.data.yuv420.yuv, stream.data.yuv420.length_yuv); - */ - } - - /* call application callback */ - _MMCAMCORDER_LOCK_VSTREAM_CALLBACK(hcamcorder); - if (hcamcorder->vstream_cb) { - hcamcorder->vstream_cb(&stream, hcamcorder->vstream_cb_param); - - for (i = 0 ; i < TBM_SURF_PLANE_MAX && stream.bo[i] ; i++) { - tbm_bo_map(stream.bo[i], TBM_DEVICE_CPU, TBM_OPTION_READ|TBM_OPTION_WRITE); - tbm_bo_unmap(stream.bo[i]); - } - } - - _MMCAMCORDER_UNLOCK_VSTREAM_CALLBACK(hcamcorder); - - /* unmap memory */ - if (mapinfo.data) - gst_memory_unmap(memory, &mapinfo); - if (_mmcamcorder_is_encoded_preview_pixel_format(sc->info_image->preview_format)) - gst_memory_unref(memory); + /* The first H.264 frame should not be skipped for vstream cb. */ + if (hcamcorder->state < MM_CAMCORDER_STATE_PREPARE && + sc->info_image->preview_format != MM_PIXEL_FORMAT_ENCODED_H264) { + _mmcam_dbg_warn("Not ready for stream callback"); + return GST_PAD_PROBE_OK; } - return GST_PAD_PROBE_OK; + if (_mmcamcorder_invoke_video_stream_cb((MMHandleType)hcamcorder, buffer, TRUE)) + return GST_PAD_PROBE_OK; + else + return GST_PAD_PROBE_DROP; } diff --git a/src/mm_camcorder_internal.c b/src/mm_camcorder_internal.c index aefe75a..fd26f73 100644 --- a/src/mm_camcorder_internal.c +++ b/src/mm_camcorder_internal.c @@ -140,6 +140,7 @@ static gint __mmcamcorder_init_handle(mmf_camcorder_t **hcamcorder, int device_t g_mutex_init(&(new_handle->mtsafe).vstream_cb_lock); g_mutex_init(&(new_handle->mtsafe).astream_cb_lock); g_mutex_init(&(new_handle->mtsafe).mstream_cb_lock); + g_mutex_init(&(new_handle->mtsafe).vedecision_cb_lock); #ifdef _MMCAMCORDER_MM_RM_SUPPORT g_mutex_init(&(new_handle->mtsafe).resource_lock); #endif /* _MMCAMCORDER_MM_RM_SUPPORT */ @@ -247,6 +248,7 @@ static void __mmcamcorder_deinit_handle(mmf_camcorder_t *hcamcorder) g_mutex_clear(&(hcamcorder->mtsafe).vstream_cb_lock); g_mutex_clear(&(hcamcorder->mtsafe).astream_cb_lock); g_mutex_clear(&(hcamcorder->mtsafe).mstream_cb_lock); + g_mutex_clear(&(hcamcorder->mtsafe).vedecision_cb_lock); #ifdef _MMCAMCORDER_MM_RM_SUPPORT g_mutex_clear(&(hcamcorder->mtsafe).resource_lock); #endif /* _MMCAMCORDER_MM_RM_SUPPORT */ @@ -1894,6 +1896,32 @@ int _mmcamcorder_set_video_capture_callback(MMHandleType handle, mm_camcorder_vi return MM_ERROR_NONE; } + +int _mmcamcorder_set_video_encode_decision_callback(MMHandleType handle, mm_camcorder_video_encode_decision_callback callback, void *user_data) +{ + mmf_camcorder_t *hcamcorder = MMF_CAMCORDER(handle); + + _mmcam_dbg_log(""); + + mmf_return_val_if_fail(hcamcorder, MM_ERROR_CAMCORDER_NOT_INITIALIZED); + + if (callback == NULL) + _mmcam_dbg_warn("Video Encode Decision Callback is disabled, because application sets it to NULL"); + + if (!_MMCAMCORDER_TRYLOCK_VEDECISION_CALLBACK(hcamcorder)) { + _mmcam_dbg_warn("Application's video encode decision callback is running now"); + return MM_ERROR_CAMCORDER_INVALID_CONDITION; + } + + hcamcorder->vedecision_cb = callback; + hcamcorder->vedecision_cb_param = user_data; + + _MMCAMCORDER_UNLOCK_VEDECISION_CALLBACK(hcamcorder); + + return MM_ERROR_NONE; +} + + int _mmcamcorder_get_current_state(MMHandleType handle) { mmf_camcorder_t *hcamcorder = MMF_CAMCORDER(handle); diff --git a/src/mm_camcorder_videorec.c b/src/mm_camcorder_videorec.c index 8ea4758..79aeab4 100644 --- a/src/mm_camcorder_videorec.c +++ b/src/mm_camcorder_videorec.c @@ -66,7 +66,7 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) _MMCamcorderImageInfo *info_image = NULL; _MMCamcorderVideoInfo *info_video = NULL; _MMCamcorderGstElement *element = NULL; - GstClockTime diff = 0; /* nsec */ + GstClockTime current_ts = 0; /* nsec */ mmf_return_val_if_fail(hcamcorder, FALSE); mmf_return_val_if_fail(MMF_CAMCORDER_SUBCONTEXT(hcamcorder), FALSE); @@ -93,6 +93,7 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) _mmcam_dbg_log("GST_BUFFER_FLAG_DELTA_UNIT is set : %d", GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT)); */ + current_ts = GST_BUFFER_PTS(buffer); if (info_video->is_first_frame) { /* check first I frame for H.264 stream */ @@ -110,7 +111,7 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) clock = GST_ELEMENT_CLOCK(element[_MMCAMCORDER_AUDIOSRC_SRC].gst); if (clock) { gst_object_ref(clock); - info_video->base_video_ts = GST_BUFFER_PTS(buffer) - (gst_clock_get_time(clock) - \ + info_video->base_video_ts = current_ts - (gst_clock_get_time(clock) - \ GST_ELEMENT(element[_MMCAMCORDER_ENCSINK_SRC].gst)->base_time); gst_object_unref(clock); } @@ -123,12 +124,19 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) g_cond_signal(&hcamcorder->task_thread_cond); g_mutex_unlock(&hcamcorder->task_thread_lock); } - info_video->base_video_ts = GST_BUFFER_PTS(buffer); + info_video->base_video_ts = current_ts; + } + } else { + if (_mmcamcorder_invoke_video_stream_cb(handle, buffer, FALSE) == FALSE) { + /*_mmcam_dbg_warn("do not push buffer to encode by app's return value");*/ + /* increase base video timestamp by frame duration, + it will remove delay of dropped buffer when play recorded file. */ + info_video->base_video_ts += current_ts - info_video->last_video_ts; + goto _VIDEO_PUSH_BUFFER_DONE; } } - GST_BUFFER_PTS(buffer) = GST_BUFFER_PTS(buffer) - info_video->base_video_ts; - GST_BUFFER_DTS(buffer) = GST_BUFFER_PTS(buffer); + GST_BUFFER_DTS(buffer) = GST_BUFFER_PTS(buffer) = current_ts - info_video->base_video_ts; /*_mmcam_dbg_log("buffer %p, timestamp %"GST_TIME_FORMAT, buffer, GST_TIME_ARGS(GST_BUFFER_PTS(buffer)));*/ @@ -142,6 +150,9 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) /*_mmcam_dbg_log("push buffer result : 0x%x", ret);*/ +_VIDEO_PUSH_BUFFER_DONE: + info_video->last_video_ts = current_ts; + if (info_video->is_first_frame) { info_video->is_first_frame = FALSE; @@ -157,8 +168,7 @@ gboolean _mmcamcorder_video_push_buffer(void *handle, GstBuffer *buffer) if (info_video->record_dual_stream == FALSE && info_video->fps > _MMCAMCORDER_FRAME_PASS_MIN_FPS) { if (info_video->prev_preview_ts != 0) { - diff = GST_BUFFER_PTS(buffer) - info_video->prev_preview_ts; - if (diff < _MMCAMCORDER_MIN_TIME_TO_PASS_FRAME) { + if (GST_BUFFER_PTS(buffer) - info_video->prev_preview_ts < _MMCAMCORDER_MIN_TIME_TO_PASS_FRAME) { _mmcam_dbg_log("it's too fast. drop frame..."); return FALSE; } @@ -220,10 +230,11 @@ int _mmcamcorder_create_recorder_pipeline(MMHandleType handle) MMCAM_AUDIO_DISABLE, &sc->audio_disable, NULL); - _mmcam_dbg_log("MMCAM_AUDIO_DISABLE %d, is_modified_rate %d", - sc->audio_disable, sc->is_modified_rate); + _mmcam_dbg_log("MMCAM_AUDIO_DISABLE %d, is_modified_rate %d, ved_cb %p", + sc->audio_disable, sc->is_modified_rate, hcamcorder->vedecision_cb); - sc->audio_disable |= sc->is_modified_rate; + if (sc->is_modified_rate || hcamcorder->vedecision_cb) + sc->audio_disable = TRUE; if (sc->audio_disable == FALSE) { /* create audiosrc bin */ -- 2.34.1