From: Jeongmo Yang Date: Thu, 31 Mar 2022 09:01:08 +0000 (+0900) Subject: Support zero-copy with encoded format X-Git-Tag: submit/tizen/20220427.095935~1 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=55657128526af525b8db0d2a43653921060b438a;p=platform%2Fcore%2Fmultimedia%2Flibmm-camcorder.git Support zero-copy with encoded format - Additional change : Clean up code for _mmcamcorder_invoke_video_stream_cb(). [Version] 0.10.256 [Issue Type] New feature Change-Id: Ib582d92dad9eacf8e2224e9d53baf49bf0db8020 Signed-off-by: Jeongmo Yang --- diff --git a/packaging/libmm-camcorder.spec b/packaging/libmm-camcorder.spec index 30cd40c..0f70e33 100755 --- a/packaging/libmm-camcorder.spec +++ b/packaging/libmm-camcorder.spec @@ -1,6 +1,6 @@ Name: libmm-camcorder Summary: Camera and recorder library -Version: 0.10.255 +Version: 0.10.256 Release: 0 Group: Multimedia/Libraries License: Apache-2.0 diff --git a/src/include/mm_camcorder_util.h b/src/include/mm_camcorder_util.h index d065bfa..6165170 100644 --- a/src/include/mm_camcorder_util.h +++ b/src/include/mm_camcorder_util.h @@ -302,7 +302,7 @@ gboolean _mmcamcorder_send_message(MMHandleType handle, _MMCamcorderMsgItem *dat void _mmcamcorder_remove_message_all(MMHandleType handle); /* Pixel format */ -int _mmcamcorder_get_pixel_format(GstCaps *pad); +int _mmcamcorder_get_pixel_format(GstCaps *caps, gboolean is_preview); int _mmcamcorder_get_pixtype(unsigned int fourcc); unsigned int _mmcamcorder_get_fourcc(int pixtype, int codectype, int use_zero_copy_format); MMPixelFormatType _mmcamcorder_get_pixel_format2(GstCameraControlImageFormat img_fmt); diff --git a/src/mm_camcorder_gstcommon.c b/src/mm_camcorder_gstcommon.c index aae8ad6..6dc8828 100755 --- a/src/mm_camcorder_gstcommon.c +++ b/src/mm_camcorder_gstcommon.c @@ -156,23 +156,34 @@ static guint32 _mmcamcorder_convert_fourcc_string_to_value(const gchar* format_n static bool __mmcamcorder_find_max_resolution(MMHandleType handle, gint *max_width, gint *max_height); #endif /* _MMCAMCORDER_PRODUCT_TV */ -static gboolean __mmcamcorder_set_stream_data_tbm(MMCamcorderVideoStreamDataType *stream, tbm_surface_info_s *ts_info); -static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMapInfo *map_info); +static gboolean __mmcamcorder_set_stream_data(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMemory *memory); +static gboolean __mmcamcorder_set_stream_data_zero_copy(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMemory *memory); /*======================================================================================= | FUNCTION DEFINITIONS | =======================================================================================*/ -static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMapInfo *map_info) +static gboolean __mmcamcorder_set_stream_data(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMemory *memory) { + gboolean ret = TRUE; + GstMapInfo map_info; + mmf_return_val_if_fail(buffer, FALSE); - mmf_return_val_if_fail(map_info, FALSE); mmf_return_val_if_fail(stream, FALSE); + memset(&map_info, 0x0, sizeof(GstMapInfo)); + + if (!gst_memory_map(memory, &map_info, GST_MAP_READWRITE)) { + MMCAM_LOG_ERROR("map failed for memory[%p]", memory); + return FALSE; + } + + stream->length_total = gst_memory_get_sizes(memory, NULL, NULL); + switch (stream->format) { case MM_PIXEL_FORMAT_NV12: /* fall through */ case MM_PIXEL_FORMAT_NV21: stream->data_type = MM_CAM_STREAM_DATA_YUV420SP; - stream->data.yuv420sp.y = map_info->data; + stream->data.yuv420sp.y = map_info.data; stream->data.yuv420sp.length_y = stream->width * stream->height; stream->data.yuv420sp.uv = stream->data.yuv420sp.y + stream->data.yuv420sp.length_y; stream->data.yuv420sp.length_uv = stream->data.yuv420sp.length_y >> 1; @@ -185,7 +196,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_I420: stream->data_type = MM_CAM_STREAM_DATA_YUV420P; - stream->data.yuv420p.y = map_info->data; + stream->data.yuv420p.y = map_info.data; stream->data.yuv420p.length_y = stream->width * stream->height; stream->data.yuv420p.u = stream->data.yuv420p.y + stream->data.yuv420p.length_y; stream->data.yuv420p.length_u = stream->data.yuv420p.length_y >> 2; @@ -200,7 +211,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_422P: stream->data_type = MM_CAM_STREAM_DATA_YUV422P; - stream->data.yuv422p.y = map_info->data; + stream->data.yuv422p.y = map_info.data; stream->data.yuv422p.length_y = stream->width * stream->height; stream->data.yuv422p.u = stream->data.yuv422p.y + stream->data.yuv422p.length_y; stream->data.yuv422p.length_u = stream->data.yuv422p.length_y >> 1; @@ -217,7 +228,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_UYVY: /* fall through */ case MM_PIXEL_FORMAT_ITLV_JPEG_UYVY: stream->data_type = MM_CAM_STREAM_DATA_YUV422; - stream->data.yuv422.yuv = map_info->data; + stream->data.yuv422.yuv = map_info.data; stream->data.yuv422.length_yuv = stream->length_total; stream->stride[0] = stream->width << 1; stream->elevation[0] = stream->height; @@ -229,7 +240,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_ENCODED_VP8: /* fall through */ case MM_PIXEL_FORMAT_ENCODED_VP9: /* fall through */ stream->data_type = MM_CAM_STREAM_DATA_ENCODED; - stream->data.encoded.data = map_info->data; + stream->data.encoded.data = map_info.data; stream->data.encoded.length_data = stream->length_total; stream->data.encoded.is_delta_frame = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); stream->num_planes = 1; @@ -237,7 +248,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_INVZ: stream->data_type = MM_CAM_STREAM_DATA_DEPTH; - stream->data.depth.data = map_info->data; + stream->data.depth.data = map_info.data; stream->data.depth.length_data = stream->length_total; stream->stride[0] = stream->width << 1; stream->elevation[0] = stream->height; @@ -247,7 +258,7 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT case MM_PIXEL_FORMAT_RGBA: /* fall through */ case MM_PIXEL_FORMAT_ARGB: stream->data_type = MM_CAM_STREAM_DATA_RGB; - stream->data.rgb.data = map_info->data; + stream->data.rgb.data = map_info.data; stream->data.rgb.length_data = stream->length_total; stream->stride[0] = stream->width << 2; stream->elevation[0] = stream->height; @@ -256,38 +267,103 @@ static gboolean __mmcamcorder_set_stream_data_normal(MMCamcorderVideoStreamDataT default: MMCAM_LOG_ERROR("unsupported format[%d]", stream->format); - return FALSE; + ret = FALSE; + break; } - return TRUE; + gst_memory_unmap(memory, &map_info); + + return ret; } -static gboolean __mmcamcorder_set_stream_data_tbm(MMCamcorderVideoStreamDataType *stream, tbm_surface_info_s *ts_info) +static gboolean __mmcamcorder_set_stream_data_zero_copy(MMCamcorderVideoStreamDataType *stream, GstBuffer *buffer, GstMemory *memory) { - mmf_return_val_if_fail(ts_info, FALSE); + int i = 0; + int num_bos = 0; + int tbm_ret = TBM_SURFACE_ERROR_NONE; + tbm_bo_handle bo_handle = {NULL, }; + tbm_surface_info_s ts_info; + mmf_return_val_if_fail(stream, FALSE); + mmf_return_val_if_fail(buffer, FALSE); + mmf_return_val_if_fail(memory, FALSE); + + stream->internal_buffer = buffer; + + if (_mmcamcorder_is_encoded_preview_pixel_format(stream->format)) { + stream->bo[0] = gst_tizen_memory_get_bos(memory, 0); + + bo_handle = tbm_bo_get_handle(stream->bo[0], TBM_DEVICE_CPU); + if (!bo_handle.ptr) { + MMCAM_LOG_ERROR("tbm_bo_get_handle failed[bo:%p,memory:%p]", stream->bo[0], memory); + return FALSE; + } + + stream->data_type = MM_CAM_STREAM_DATA_ENCODED; + stream->num_planes = 1; + stream->stride[0] = stream->width; + stream->elevation[0] = stream->height; + stream->data.encoded.data = bo_handle.ptr; + stream->length_total = gst_memory_get_sizes(memory, NULL, NULL); + stream->data.encoded.length_data = stream->length_total; + stream->data.encoded.is_delta_frame = GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); + + MMCAM_LOG_VERBOSE("[ENCODED] length[%u], is_delta[%d]", + stream->data.encoded.length_data, stream->data.encoded.is_delta_frame); + + return TRUE; + } + + memset(&ts_info, 0x0, sizeof(tbm_surface_info_s)); + + tbm_ret = tbm_surface_get_info((tbm_surface_h)gst_tizen_memory_get_surface(memory), &ts_info); + if (tbm_ret != TBM_SURFACE_ERROR_NONE) { + MMCAM_LOG_ERROR("get tbm surface info failed[0x%x]", tbm_ret); + return FALSE; + } + + stream->length_total = ts_info.size; + for (i = 0 ; i < ts_info.num_planes ; i++) { + stream->stride[i] = ts_info.planes[i].stride; + stream->elevation[i] = ts_info.planes[i].size / ts_info.planes[i].stride; + MMCAM_LOG_VERBOSE(" plane[%d] %dx%d", i, stream->stride[i], stream->elevation[i]); + } + + num_bos = gst_tizen_memory_get_num_bos(memory); + for (i = 0 ; i < num_bos ; i++) + stream->bo[i] = gst_tizen_memory_get_bos(memory, i); + switch (stream->format) { case MM_PIXEL_FORMAT_NV12: /* fall through */ case MM_PIXEL_FORMAT_NV21: stream->data_type = MM_CAM_STREAM_DATA_YUV420SP; stream->num_planes = 2; - stream->data.yuv420sp.y = ts_info->planes[0].ptr; - stream->data.yuv420sp.length_y = ts_info->planes[0].size; - stream->data.yuv420sp.uv = ts_info->planes[1].ptr; - stream->data.yuv420sp.length_uv = ts_info->planes[1].size; + stream->data.yuv420sp.y = ts_info.planes[0].ptr; + stream->data.yuv420sp.length_y = ts_info.planes[0].size; + stream->data.yuv420sp.uv = ts_info.planes[1].ptr; + stream->data.yuv420sp.length_uv = ts_info.planes[1].size; + + MMCAM_LOG_VERBOSE("[420SP] 0[%p,%u], 1[%p,%u]", + stream->data.yuv420sp.y, stream->data.yuv420sp.length_y, + stream->data.yuv420sp.uv, stream->data.yuv420sp.length_uv); break; case MM_PIXEL_FORMAT_I420: stream->data_type = MM_CAM_STREAM_DATA_YUV420P; stream->num_planes = 3; - stream->data.yuv420p.y = ts_info->planes[0].ptr; - stream->data.yuv420p.length_y = ts_info->planes[0].size; - stream->data.yuv420p.u = ts_info->planes[1].ptr; - stream->data.yuv420p.length_u = ts_info->planes[1].size; - stream->data.yuv420p.v = ts_info->planes[2].ptr; - stream->data.yuv420p.length_v = ts_info->planes[2].size; + stream->data.yuv420p.y = ts_info.planes[0].ptr; + stream->data.yuv420p.length_y = ts_info.planes[0].size; + stream->data.yuv420p.u = ts_info.planes[1].ptr; + stream->data.yuv420p.length_u = ts_info.planes[1].size; + stream->data.yuv420p.v = ts_info.planes[2].ptr; + stream->data.yuv420p.length_v = ts_info.planes[2].size; + + MMCAM_LOG_VERBOSE("[420P] 0[%p,%u], 1[%p,%u], 2[%p,%u]", + stream->data.yuv420p.y, stream->data.yuv420p.length_y, + stream->data.yuv420p.u, stream->data.yuv420p.length_u, + stream->data.yuv420p.v, stream->data.yuv420p.length_v); break; default: @@ -302,18 +378,13 @@ static gboolean __mmcamcorder_set_stream_data_tbm(MMCamcorderVideoStreamDataType gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstSample *sample, gboolean is_preview, int stream_id) { int i = 0; - int num_bos = 0; gboolean ret_cb = TRUE; mmf_camcorder_t *hcamcorder = MMF_CAMCORDER(handle); _MMCamcorderSubContext *sc = NULL; MMCamcorderVideoStreamDataType stream; - tbm_surface_h t_surface = NULL; - tbm_surface_info_s ts_info; - GstBuffer *buffer = NULL; GstMemory *memory = NULL; - GstMapInfo map_info; GstCaps *caps = NULL; GstStructure *structure = NULL; @@ -326,8 +397,6 @@ gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstSample *sam sc = MMF_CAMCORDER_SUBCONTEXT(hcamcorder); mmf_return_val_if_fail(sc, FALSE); - /* clear data structure */ - memset(&map_info, 0x0, sizeof(GstMapInfo)); memset(&stream, 0x0, sizeof(MMCamcorderVideoStreamDataType)); caps = gst_sample_get_caps(sample); @@ -336,19 +405,13 @@ gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstSample *sam structure = gst_caps_get_structure(caps, 0); mmf_return_val_if_fail(structure, FALSE); - /* set format and resolution */ + stream.format = _mmcamcorder_get_pixel_format(caps, TRUE); gst_structure_get(structure, "width", G_TYPE_INT, &stream.width, "height", G_TYPE_INT, &stream.height, NULL); - stream.format = _mmcamcorder_get_pixel_format(caps); - if (_mmcamcorder_is_encoded_preview_pixel_format(stream.format)) { - memory = gst_buffer_get_all_memory(buffer); - stream.internal_buffer = buffer; - } else { - memory = gst_buffer_peek_memory(buffer, 0); - } + memory = gst_buffer_peek_memory(buffer, 0); if (!memory) { MMCAM_LOG_ERROR("GstMemory get failed from buffer %p", buffer); return FALSE; @@ -367,36 +430,11 @@ gboolean _mmcamcorder_invoke_video_stream_cb(MMHandleType handle, GstSample *sam stream.focus_state, stream.facing_direction, stream.flip, stream.rotation); } - /* set zero-copy related information */ - if (hcamcorder->use_zero_copy_format) { - t_surface = (tbm_surface_h)gst_tizen_memory_get_surface(memory); - - if (tbm_surface_get_info(t_surface, &ts_info) != TBM_SURFACE_ERROR_NONE) { - MMCAM_LOG_ERROR("failed to get tbm surface[%p] info", t_surface); - goto _INVOKE_VIDEO_STREAM_CB_DONE; - } - - /* set bo, stride and elevation */ - num_bos = gst_tizen_memory_get_num_bos(memory); - for (i = 0 ; i < num_bos ; i++) - stream.bo[i] = gst_tizen_memory_get_bos(memory, i); - - for (i = 0 ; i < ts_info.num_planes ; i++) { - stream.stride[i] = ts_info.planes[i].stride; - stream.elevation[i] = ts_info.planes[i].size / ts_info.planes[i].stride; - MMCAM_LOG_VERBOSE(" plane[%d] %dx%d", i, stream.stride[i], stream.elevation[i]); - } - - stream.length_total = ts_info.size; - stream.internal_buffer = buffer; - - if (!__mmcamcorder_set_stream_data_tbm(&stream, &ts_info)) + if (gst_is_tizen_memory(memory)) { + if (!__mmcamcorder_set_stream_data_zero_copy(&stream, buffer, memory)) goto _INVOKE_VIDEO_STREAM_CB_DONE; } else { - stream.length_total = gst_memory_get_sizes(memory, NULL, NULL); - - if (!gst_memory_map(memory, &map_info, GST_MAP_READWRITE) || - !__mmcamcorder_set_stream_data_normal(&stream, buffer, &map_info)) + if (!__mmcamcorder_set_stream_data(&stream, buffer, memory)) goto _INVOKE_VIDEO_STREAM_CB_DONE; } @@ -425,12 +463,6 @@ _INVOKE_VIDEO_STREAM_CB_DONE: tbm_bo_unmap(stream.bo[i]); } - if (map_info.data) - gst_memory_unmap(memory, &map_info); - - if (_mmcamcorder_is_encoded_preview_pixel_format(stream.format)) - gst_memory_unref(memory); - return ret_cb; } diff --git a/src/mm_camcorder_stillshot.c b/src/mm_camcorder_stillshot.c index b2ac5c2..a7834f7 100644 --- a/src/mm_camcorder_stillshot.c +++ b/src/mm_camcorder_stillshot.c @@ -1341,7 +1341,7 @@ static void __mmcamcorder_image_capture_cb(GstElement *element, GstSample *sampl memset((void *)&scrnail, 0x0, sizeof(MMCamcorderCaptureDataType)); /* Prepare main, thumbnail buffer */ - pixtype_main = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample1)); + pixtype_main = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample1), FALSE); if (pixtype_main == MM_PIXEL_FORMAT_INVALID) { MMCAM_LOG_ERROR("Unsupported pixel type"); MMCAM_SEND_MESSAGE(hcamcorder, MM_MESSAGE_CAMCORDER_ERROR, MM_ERROR_CAMCORDER_INTERNAL); @@ -1377,7 +1377,7 @@ static void __mmcamcorder_image_capture_cb(GstElement *element, GstSample *sampl if (sample3 && mapinfo3.data && mapinfo3.size != 0) { MMCAM_LOG_INFO("Screennail (sample3=%p,size=%zu)", sample3, mapinfo3.size); - pixtype_scrnl = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample3)); + pixtype_scrnl = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample3), FALSE); __mmcamcorder_get_capture_data_from_buffer(&scrnail, pixtype_scrnl, sample3); /* Set screennail attribute for application */ @@ -1400,7 +1400,7 @@ static void __mmcamcorder_image_capture_cb(GstElement *element, GstSample *sampl /* Thumbnail image buffer */ if (sample2 && mapinfo2.data && (mapinfo2.size != 0)) { MMCAM_LOG_INFO("Thumbnail (buffer2=%p)", gst_sample_get_buffer(sample2)); - pixtype_thumb = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample2)); + pixtype_thumb = _mmcamcorder_get_pixel_format(gst_sample_get_caps(sample2), FALSE); __mmcamcorder_get_capture_data_from_buffer(&thumb, pixtype_thumb, sample2); } else { MMCAM_LOG_INFO("Sample2 has wrong pointer. Not Error. (sample2 %p)", sample2); diff --git a/src/mm_camcorder_util.c b/src/mm_camcorder_util.c index 8dbea0b..b7c8852 100644 --- a/src/mm_camcorder_util.c +++ b/src/mm_camcorder_util.c @@ -1528,7 +1528,7 @@ void _mmcamcorder_remove_message_all(MMHandleType handle) } -int _mmcamcorder_get_pixel_format(GstCaps *caps) +int _mmcamcorder_get_pixel_format(GstCaps *caps, gboolean is_preview) { gchar *caps_string = NULL; const char *media_type; @@ -1549,7 +1549,7 @@ int _mmcamcorder_get_pixel_format(GstCaps *caps) } if (!strcmp(media_type, "image/jpeg")) - return MM_PIXEL_FORMAT_ENCODED; + return (is_preview ? MM_PIXEL_FORMAT_ENCODED_MJPEG : MM_PIXEL_FORMAT_ENCODED); else if (!strcmp(media_type, "video/x-h264")) return MM_PIXEL_FORMAT_ENCODED_H264; else if (!strcmp(media_type, "video/x-jpeg")) @@ -2470,7 +2470,8 @@ int _mmcamcorder_get_audiosrc_blocksize(int samplerate, int format, int channel, gboolean _mmcamcorder_is_encoded_preview_pixel_format(int pixel_format) { - return (pixel_format == MM_PIXEL_FORMAT_ENCODED_H264 || + return (pixel_format == MM_PIXEL_FORMAT_ENCODED || + pixel_format == MM_PIXEL_FORMAT_ENCODED_H264 || pixel_format == MM_PIXEL_FORMAT_ENCODED_MJPEG || pixel_format == MM_PIXEL_FORMAT_ENCODED_VP8 || pixel_format == MM_PIXEL_FORMAT_ENCODED_VP9);