From 5ecac2f379ee90b681ba830a542e1d96020842cd Mon Sep 17 00:00:00 2001 From: Eunhye Choi Date: Wed, 25 Aug 2021 17:23:49 +0900 Subject: [PATCH] support video decoded callback - video frame could be exported if app register the video decoded cb - related api : esplusplayer_set_media_packet_video_decoded_cb esplusplayer_decoded_buffer_destroy Change-Id: I7cad9cbd50c56f65fefe2d09c1d43f7fb1082ce9 --- include/trackrenderer_capi/buffer.h | 2 +- src/CMakeLists.txt | 2 +- .../trackrenderer/core/buffer.h | 3 +- .../trackrenderer/trackrenderer.h | 5 +- src/trackrenderer.cpp | 515 +++++++----------- src/trackrenderer_capi_utils.cpp | 2 +- 6 files changed, 204 insertions(+), 325 deletions(-) diff --git a/include/trackrenderer_capi/buffer.h b/include/trackrenderer_capi/buffer.h index 0942d6f..088d736 100644 --- a/include/trackrenderer_capi/buffer.h +++ b/include/trackrenderer_capi/buffer.h @@ -70,7 +70,7 @@ typedef struct { /** * @description the scaler index,0 1 ... */ - void* scaler_index; + void* buffer_addr; } TrackRendererDecodedVideoPacket; #ifdef __cplusplus diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c67790e..eedc7c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,7 +11,7 @@ SET(ADD_LIBS SET(${fw_name}_CXXFLAGS "-Wall -Werror -std=c++11 -fPIC -Wl,-z,relro -fstack-protector -DEFL_BETA_API_SUPPORT") -SET(dependents "gstreamer-1.0 gstreamer-video-1.0" +SET(dependents "gstreamer-1.0 gstreamer-plugins-base-1.0 gstreamer-video-1.0 gstreamer-allocators-1.0" "boost" "vconf" "elementary ecore ecore-wl2" diff --git a/src/include_internal/trackrenderer/core/buffer.h b/src/include_internal/trackrenderer/core/buffer.h index d1dd828..1253189 100644 --- a/src/include_internal/trackrenderer/core/buffer.h +++ b/src/include_internal/trackrenderer/core/buffer.h @@ -28,7 +28,7 @@ struct DecodedVideoPacket { uint64_t pts = 0; uint64_t duration = 0; tbm_surface_h surface_data = nullptr; // tbm_surface - void *scaler_index = nullptr; + void *buffer_addr = nullptr; }; static constexpr int kHandoffFrameBufferNum = 2; @@ -62,7 +62,6 @@ class TbmBufferManager { public: tbm_bufmgr bufmgr = nullptr; tbm_surface_h tbm_surface[kHandoffFrameBufferNum] = {nullptr}; - int scaler_index = 0; }; // class TbmBufferManager } // namespace trackrenderer diff --git a/src/include_internal/trackrenderer/trackrenderer.h b/src/include_internal/trackrenderer/trackrenderer.h index 83f4a9b..2eacf32 100644 --- a/src/include_internal/trackrenderer/trackrenderer.h +++ b/src/include_internal/trackrenderer/trackrenderer.h @@ -260,10 +260,7 @@ class TrackRenderer : public ResourceConflictListener, static gboolean GstAudioDrmInitDataCb_(int* drmhandle, unsigned int len, unsigned char* psshdata, void* userdata); - static void GstDecodedVideoReferenceBufferCb_(GstElement* element, - GstBuffer* buffer, GstPad* pad, - void* userdata); - static void GstDecodedVideoCopyBufferCb_(GstElement* element, + static void GstDecodedVideoBufferCb_(GstElement* element, GstBuffer* buffer, GstPad* pad, void* userdata); static AttributesByElement InitAttributeByElementType_(); diff --git a/src/trackrenderer.cpp b/src/trackrenderer.cpp index 793fa7f..1633ad8 100644 --- a/src/trackrenderer.cpp +++ b/src/trackrenderer.cpp @@ -13,13 +13,10 @@ #include #include #include -#include "system_info.h" +#include +#include +#include -// performance logs in popup log file -#include -#include -#include -// end #include "trackrenderer/core/gst_utils.h" #include "trackrenderer/core/gstobject_guard.h" #include "trackrenderer/core/track_util.h" @@ -208,140 +205,174 @@ inline bool IsDecoderElementNecessary(const std::string& mimetype) { inline bool IsVideoDecodedBufferNeeded(DecodedVideoFrameBufferType& type) { return (type != DecodedVideoFrameBufferType::kNone); } -constexpr int kVideoBufferPlaneMax = 4; -struct VideoStreamDataType { - tbm_format format = TBM_FORMAT_NV12; //< image format - int width = 0; //< width of video buffer - int height = 0; //< height of video buffer - tbm_bo bo[kVideoBufferPlaneMax] = {nullptr}; //< TBM buffer object - void* internal_buffer = nullptr; //< Internal buffer pointer - int stride[kVideoBufferPlaneMax] = {0}; //< stride of plane - int elevation[kVideoBufferPlaneMax] = {0}; //< elevation of plane -}; - -bool CopySwCodec(GstBuffer* decodedBuffer, const VideoStreamDataType& stream) { - tbm_bo bo_Y = stream.bo[GST_VIDEO_COMP_Y]; - tbm_bo bo_C = stream.bo[GST_VIDEO_COMP_U]; - - tbm_bo_handle ap_bo_handle = { - nullptr, - }; - GstStructure* s = GST_STRUCTURE_CAST(gst_mini_object_get_qdata( - GST_MINI_OBJECT(decodedBuffer), g_quark_from_static_string("tbm_bo"))); - if (s) { - if (!gst_structure_get(s, "tbm_bo_hnd", G_TYPE_POINTER, &ap_bo_handle, - NULL)) { - TRACKRENDERER_ERROR("Buffer don't have tbm_bo_hnd structure"); - return false; - } - } else { - TRACKRENDERER_ERROR("Buffer don't have tbm_bo structure"); - return false; +uint32_t ConvertToTbmFormat(const gchar *data) { + uint32_t format = 0; + + switch GST_MAKE_FOURCC(data[0], data[1], data[2], data[3]) { + case GST_MAKE_FOURCC('S', '4', '2', '0'): + case GST_MAKE_FOURCC('I', '4', '2', '0'): + format = TBM_FORMAT_YUV420; + break; + case GST_MAKE_FOURCC('B', 'G', 'R', 'A'): + case GST_MAKE_FOURCC('B', 'G', 'R', 'x'): + case GST_MAKE_FOURCC('S', 'R', '3', '2'): + format = TBM_FORMAT_ARGB8888; + break; + default: + TRACKRENDERER_ERROR("Not supported type (%c%c%c%c)", + data[0], data[1], data[2], data[3]); + break; } + return format; +} - tbm_bo_handle bo_handle_Y = - tbm_bo_map(bo_Y, TBM_DEVICE_CPU, TBM_OPTION_WRITE); - if (!bo_handle_Y.ptr) { - TRACKRENDERER_ERROR("TBM get error : bo_handle_Y.ptr is NULL"); - return false; +constexpr int kMaxPlane = 4; +tbm_surface_h CreateTbmSurfaceWithBuffer(GstMemory* mem, GstPad* pad) { + + int width = 0; + int height = 0; + uint32_t bo_format = 0; + const gchar* string_format = nullptr; + GstStructure* structure = nullptr; + GstMapInfo mapinfo = GST_MAP_INFO_INIT; + + TRACKRENDERER_ENTER; + + bool is_mapped = gst_memory_map(mem, &mapinfo, GST_MAP_READWRITE); + if (!is_mapped) { + TRACKRENDERER_ERROR("gst_memory_map fail"); + return nullptr; } - for (int i = 0; i < stream.height; i++) { - memcpy((unsigned char*)(bo_handle_Y.ptr) + - (i * stream.stride[GST_VIDEO_COMP_Y]), - (unsigned char*)(ap_bo_handle.ptr) + (i * stream.width), - stream.width); + BOOST_SCOPE_EXIT(&mem, &mapinfo) { + gst_memory_unmap(mem, &mapinfo); } - tbm_bo_unmap(bo_Y); - - tbm_bo_handle bo_handle_C = - tbm_bo_map(bo_C, TBM_DEVICE_CPU, TBM_OPTION_WRITE); - if (!bo_handle_C.ptr) { - TRACKRENDERER_ERROR("TBM get error : bo_handle_C.ptr is NULL"); - return false; + BOOST_SCOPE_EXIT_END + if (!mapinfo.data) { + TRACKRENDERER_ERROR("data pointer is wrong"); + return nullptr; } - int data_size = stream.width * stream.height; - for (int i = 0; i < stream.height / 2; i++) { - memcpy((unsigned char*)(bo_handle_C.ptr) + - (i * stream.stride[GST_VIDEO_COMP_U]), - (unsigned char*)(ap_bo_handle.ptr) + data_size + (i * stream.width), - stream.width); + + auto caps = gstguard::make_guard(gst_pad_get_current_caps(pad)); + if (!caps.get()) { + TRACKRENDERER_ERROR("fail to get caps"); + return nullptr; } - tbm_bo_unmap(bo_C); - return true; -} + structure = gst_caps_get_structure(caps.get(), 0); + gst_structure_get_int(structure, "width", &width); + gst_structure_get_int(structure, "height", &height); + string_format = gst_structure_get_string(structure, "format"); -bool CopyHwCodec(const VideoStreamDataType& stream, uintptr_t y_viraddr, - uintptr_t c_viraddr, int y_linesize, int c_linesize) { - tbm_bo bo_Y = stream.bo[GST_VIDEO_COMP_Y]; - tbm_bo bo_C = stream.bo[GST_VIDEO_COMP_U]; + bo_format = internal::ConvertToTbmFormat(string_format); + if (!bo_format) { + TRACKRENDERER_ERROR("not supported video format"); + return nullptr; + } - tbm_bo_handle bo_handle_Y = - tbm_bo_map(bo_Y, TBM_DEVICE_CPU, TBM_OPTION_WRITE); - if (!bo_handle_Y.ptr) { - TRACKRENDERER_ERROR("TBM get error : bo_handle_Y.ptr is NULL"); - return false; + tbm_surface_h tbm_surf = + tbm_surface_create(width, height, bo_format); + if (!tbm_surf) { + TRACKRENDERER_ERROR("fail to create tbm surface"); + return nullptr; } - for (int i = 0; i < stream.height; i++) { - memcpy((unsigned char*)(bo_handle_Y.ptr) + - (i * stream.stride[GST_VIDEO_COMP_Y]), - (void *)y_viraddr, stream.width); - y_viraddr += y_linesize; + tbm_surface_info_s info; + int ret = tbm_surface_get_info(tbm_surf, &info); + if (ret != TBM_SURFACE_ERROR_NONE) { + TRACKRENDERER_ERROR("fail to get tbm surface info"); + tbm_surface_destroy(tbm_surf); + return nullptr; } - tbm_bo_unmap(bo_Y); - tbm_bo_handle bo_handle_C = - tbm_bo_map(bo_C, TBM_DEVICE_CPU, TBM_OPTION_WRITE); - if (!bo_handle_C.ptr) { - TRACKRENDERER_ERROR("TBM get error : bo_handle_C.ptr is NULL"); - return false; + tbm_bo bo = tbm_surface_internal_get_bo(tbm_surf, 0); + uint32_t bo_size = tbm_bo_size(bo); + tbm_bo_handle thandle = tbm_bo_map(bo, TBM_DEVICE_CPU, TBM_OPTION_WRITE); + if (!thandle.ptr) { + TRACKRENDERER_ERROR("thandle pointer is wrong"); + tbm_surface_destroy(tbm_surf); + return nullptr; } - for (int i = 0; i < stream.height / 2; i++) { - memcpy((unsigned char*)(bo_handle_C.ptr) + - (i * stream.stride[GST_VIDEO_COMP_U]), - (void *)c_viraddr, stream.width); - c_viraddr += c_linesize; + BOOST_SCOPE_EXIT(&thandle, &bo) { + if (thandle.ptr) { + tbm_bo_unmap(bo); + } } - tbm_bo_unmap(bo_C); - - return true; -} + BOOST_SCOPE_EXIT_END -tbm_surface_h CreateTbmSurfaceWithBos(VideoStreamDataType& stream) { - unsigned int bo_num = 0; - for (int i = 0; i < internal::kVideoBufferPlaneMax; i++) { - if (stream.bo[i]) bo_num++; - } - tbm_surface_info_s surface_info; - memset(&surface_info, 0, sizeof(surface_info)); - surface_info.width = stream.width; - surface_info.height = stream.height; - surface_info.format = TBM_FORMAT_NV12; - surface_info.bpp = tbm_surface_internal_get_bpp(surface_info.format); - surface_info.num_planes = - tbm_surface_internal_get_num_planes(surface_info.format); - for (unsigned int i = 0; i < surface_info.num_planes; i++) { - surface_info.planes[i].stride = stream.stride[i]; - surface_info.planes[i].size = stream.stride[i] * stream.elevation[i]; - if (i < bo_num) { - surface_info.planes[i].offset = 0; - } else { - surface_info.planes[i].offset = - surface_info.planes[i - 1].offset + surface_info.planes[i - 1].size; + int plane_stride[kMaxPlane] = { 0, }; + int plane_elevation[kMaxPlane] = { 0, }; + int src_stride[kMaxPlane] = { 0, }; + int src_offset[kMaxPlane] = { 0, }; + int dest_offset[kMaxPlane] = { 0, }; + int i = 0; + int j = 0; + int k = 0; + unsigned char *src = NULL; + unsigned char *dest = NULL; + + TRACKRENDERER_ERROR("width %d, height %d, bo_size %d", width, height, bo_size); + + if (bo_format == TBM_FORMAT_YUV420) { + plane_stride[0] = info.planes[0].stride; + plane_elevation[0] = info.planes[0].size / info.planes[0].stride; + plane_stride[1] = info.planes[1].stride; + plane_elevation[1] = info.planes[1].size / info.planes[1].stride; + plane_stride[2] = info.planes[2].stride; + plane_elevation[2] = info.planes[2].size / info.planes[2].stride; + + src_stride[0] = GST_ROUND_UP_4(width); + src_stride[1] = src_stride[2] = GST_ROUND_UP_4(width >> 1); + src_offset[1] = src_stride[0] * GST_ROUND_UP_2(height); + src_offset[2] = src_offset[1] + (src_stride[1] * (GST_ROUND_UP_2(height) >> 1)); + + dest_offset[0] = 0; + dest_offset[1] = plane_stride[0] * plane_elevation[0]; + dest_offset[2] = dest_offset[1] + plane_stride[1] * plane_elevation[1]; + + for (i = 0; i < 3; i++) { + src = mapinfo.data + src_offset[i]; + dest = (unsigned char*)(thandle.ptr) + dest_offset[i]; + + if (i > 0) + k = 1; + + for (j = 0; j < height >> k; j++) { + memcpy(dest, src, width>>k); + src += src_stride[i]; + dest += plane_stride[i]; + } } - surface_info.size += surface_info.planes[i].size; - } - tbm_surface_h tbm_surf = - tbm_surface_internal_create_with_bos(&surface_info, stream.bo, bo_num); - if (!tbm_surf) { - TRACKRENDERER_ERROR("failed to create tbm surface"); - return nullptr; + } else if (bo_format == TBM_FORMAT_ARGB8888) { /* emulator */ + memcpy(thandle.ptr, mapinfo.data, bo_size); } + return tbm_surf; } +#ifdef __DEBUG__ +void DumpVideoFrame(tbm_surface_h tbm_surf) { + static int cnt = 0; + char filename[128] = {0}; + tbm_surface_info_s info; + if (tbm_surface_get_info(tbm_surf, &info)) { + TRACKRENDERER_ERROR("tsurf get_info is failed"); + return; + } + sprintf(filename, "video_frame_dump_%d", cnt); + + tbm_surface_internal_dump_start((char *)"/tmp", info.width, info.height, 1); + tbm_surface_internal_dump_buffer(tbm_surf, filename); + tbm_surface_internal_dump_end(); + + TRACKRENDERER_INFO("[0]stride : %d, offset : %d", (int)info.planes[0].stride, (int)info.planes[0].offset); + TRACKRENDERER_INFO("[1]stride : %d, offset : %d", (int)info.planes[1].stride, (int)info.planes[1].offset); + TRACKRENDERER_INFO("[2]stride : %d, offset : %d", (int)info.planes[2].stride, (int)info.planes[2].offset); + TRACKRENDERER_INFO("DUMP_OUT_IMG_%d : buffer size(%d) surf(%p) %d*%d", cnt, (int)info.size, tbm_surf, info.width, info.height); + cnt++; +} +#endif + int ToBufferTypeForSink(const DecodedVideoFrameBufferType& type) { // 0:copy, 1:reference, -1:none (refer to gstwaylnadsink) constexpr int kCopy = 0; @@ -1080,23 +1111,26 @@ bool TrackRenderer::CreateVideoPipeline_(const Track* track) { } bool TrackRenderer::CreateDecodedVideoPipeline_(const Track* track) { - TRACKRENDERER_ENTER; + TRACKRENDERER_ENTER; CreateTbmBufferManager_(track); auto caps = caps_builder_.Build(*track, internal::IsDrmEmeElementNecessary( drm_property_, track->mimetype)); CreateAppSrc_(kTrackTypeVideo, track->mimetype); - if (!pipeline_->FactoryMake(Elements::kDecVideo, (GstCaps*)caps.GetCaps_(), - GST_ELEMENT_FACTORY_TYPE_DECODER, NULL)) { + + pipeline_->FactoryMake(Elements::kParseVideo, (GstCaps*)caps.GetCaps_(), + GST_ELEMENT_FACTORY_TYPE_PARSER, NULL); + auto parse_caps = + gstguard::make_guard(pipeline_->GetSrcPadCaps(Elements::kParseVideo)); + + if (!pipeline_->FactoryMake(Elements::kDecVideo, (GstCaps*)parse_caps.get(), + GST_ELEMENT_FACTORY_TYPE_DECODER, NULL)) { const ErrorType err = ErrorType::kNotSupportedVideoCodec; eventlistener_->OnError(err); return false; } pipeline_->FactoryMake(Elements::kSinkVideo, "fakesink", "fakesink"); - if (video_pre_display_mode_) { - pipeline_->SetProperty(Elements::kSinkVideo, "accurate-resume", 1); - } pipeline_->CreateBin(Elements::kBinVideo, "videobin"); @@ -1122,8 +1156,21 @@ bool TrackRenderer::CreateDecodedVideoPipeline_(const Track* track) { Elements::kDrmVideo, Elements::kDecVideo, Elements::kSinkVideo); } else { - pipeline_->BinAdd(Elements::kBinVideo, Elements::kAppSrcVideo, - Elements::kDecVideo, Elements::kSinkVideo); + if (pipeline_->IsFactoryListType(Elements::kDecVideo, GST_ELEMENT_FACTORY_TYPE_HARDWARE)) { + TRACKRENDERER_INFO("HW codec"); + pipeline_->BinAdd(Elements::kBinVideo, Elements::kAppSrcVideo, Elements::kParseVideo, + Elements::kDecVideo, Elements::kSinkVideo); + } else { + TRACKRENDERER_INFO("SW codec"); + pipeline_->FactoryMake(Elements::kVideoQueue, "queue", NULL); + pipeline_->SetProperty(Elements::kVideoQueue, "max-size-buffers", 2); + + pipeline_->FactoryMake(Elements::kVideoConvert, "videoconvert", NULL); + pipeline_->SetProperty(Elements::kVideoConvert, "n-threads", 2); + + pipeline_->BinAdd(Elements::kBinVideo, Elements::kAppSrcVideo, Elements::kParseVideo, + Elements::kDecVideo, Elements::kVideoQueue, Elements::kVideoConvert, Elements::kSinkVideo); + } } pipeline_->BinAddSimple(Elements::kPipeline, Elements::kBinVideo); @@ -1133,33 +1180,11 @@ bool TrackRenderer::CreateDecodedVideoPipeline_(const Track* track) { pipeline_->SetProperty(Elements::kAppSrcVideo, "stream-type", GST_APP_STREAM_TYPE_SEEKABLE); - /*in case of omx seamless mode,if not render omx data directly,like copy - * omxdata to DP buffer case,need set bNoVideoOut= True .Guide from - * taing.kim@samsung.com,min.byun@samsung.com*/ - pipeline_->SetProperty(Elements::kDecVideo, "display-omxdata-direct", FALSE); - TRACKRENDERER_INFO( - "set [ %d ] to 'display-omxdata-direct' property of omx seamless " - "videodec", - FALSE); - pipeline_->SetProperty(Elements::kSinkVideo, "signal-handoffs", TRUE); - - if (decoded_buffer_type_ == DecodedVideoFrameBufferType::kCopy) { - constexpr int kTbmBoType = - 1; // 1:tbm bo, 0:CMA (refer to gstffmpegdec element, currently for - // this feature use only sw decoder(ffmpegdec) and tbm_bo) - pipeline_->SetProperty(Elements::kDecVideo, "tbm-buffer-type", kTbmBoType); - pipeline_->SignalConnect(Elements::kSinkVideo, "handoff", - G_CALLBACK(GstDecodedVideoCopyBufferCb_), this); - } else if (decoded_buffer_type_ == DecodedVideoFrameBufferType::kReference) { - constexpr int kModeYoutube360 = - 1; // 1:youtube 360, 0:usb 360, -1:normal (refer to gstomxvideo) - pipeline_->SetProperty(Elements::kDecVideo, "vr360-mode", kModeYoutube360); - pipeline_->SignalConnect(Elements::kSinkVideo, "handoff", - G_CALLBACK(GstDecodedVideoReferenceBufferCb_), - this); - } + pipeline_->SetProperty(Elements::kSinkVideo, "signal-handoffs", TRUE, "sync", TRUE, "async", TRUE); - pipeline_->SetProperty(Elements::kSinkVideo, "sync", TRUE, "async", TRUE); + pipeline_->SignalConnect(Elements::kSinkVideo, "handoff", + G_CALLBACK(GstDecodedVideoBufferCb_), + this); TRACKRENDERER_LEAVE; return true; @@ -1763,185 +1788,43 @@ void TrackRenderer::GstElementCreatedCb_(Elements element) { } } -void TrackRenderer::GstDecodedVideoReferenceBufferCb_(GstElement* element, - GstBuffer* buffer, - GstPad* pad, - void* userdata) { - GstStructure* s = GST_STRUCTURE_CAST(gst_mini_object_get_qdata( - GST_MINI_OBJECT(buffer), g_quark_from_static_string("v4l2_info"))); - if (!s) { - TRACKRENDERER_ERROR("scaler buffer don't have v4l2_info structure"); - return; - } - int width, height, y_linesize, c_linesize, y_fd, c_fd, plane_num; - gst_structure_get_int(s, "width", &width); - gst_structure_get_int(s, "height", &height); - gst_structure_get_int(s, "y_linesize", &y_linesize); - gst_structure_get_int(s, "u_linesize", &c_linesize); - gst_structure_get_int(s, "y_phyaddr", &y_fd); - gst_structure_get_int(s, "u_phyaddr", &c_fd); - gst_structure_get_int(s, "plane_num", &plane_num); - - auto trackrenderer = static_cast(userdata); - - tbm_bo bo_Y = - tbm_bo_import_fd(trackrenderer->tbm_buffer_manager_->bufmgr, y_fd); - if (bo_Y == nullptr) { - TRACKRENDERER_ERROR("[Weak Ref] tbm_bo_import_fd() error : Y bo is NULL"); - return; - } - BOOST_SCOPE_EXIT(&bo_Y) { - if (bo_Y) { - tbm_bo_unref(bo_Y); - bo_Y = nullptr; - } - } - BOOST_SCOPE_EXIT_END - - tbm_bo bo_C = - tbm_bo_import_fd(trackrenderer->tbm_buffer_manager_->bufmgr, c_fd); - if (bo_C == nullptr) { - TRACKRENDERER_ERROR("[Weak Ref] tbm_bo_import_fd() error : C bo is NULL"); - return; - } - BOOST_SCOPE_EXIT(&bo_C) { - if (bo_C) { - tbm_bo_unref(bo_C); - bo_C = nullptr; - } - } - BOOST_SCOPE_EXIT_END - - tbm_bo bo[internal::kVideoBufferPlaneMax] = {nullptr}; - bo[GST_VIDEO_COMP_Y] = bo_Y; - bo[GST_VIDEO_COMP_U] = bo_C; - - tbm_surface_info_s info; - memset(&info, 0, sizeof(info)); - info.width = width; - info.height = height; - info.format = TBM_FORMAT_NV12; - - info.bpp = tbm_surface_internal_get_bpp(info.format); - info.num_planes = tbm_surface_internal_get_num_planes(info.format); - - info.planes[GST_VIDEO_COMP_Y].stride = y_linesize; - info.planes[GST_VIDEO_COMP_Y].size = y_linesize * c_linesize; - info.size += info.planes[GST_VIDEO_COMP_Y].size; - info.planes[GST_VIDEO_COMP_U].stride = c_linesize; - info.planes[GST_VIDEO_COMP_U].size = c_linesize * c_linesize / 2; - info.size += info.planes[GST_VIDEO_COMP_U].size; - - tbm_surface_h tbm_surf = - tbm_surface_internal_create_with_bos(&info, bo, info.num_planes); - if (!tbm_surf) { - TRACKRENDERER_ERROR("failed to create tbm surface"); - return; - } +void TrackRenderer::GstDecodedVideoBufferCb_(GstElement* element, + GstBuffer* buffer, + GstPad* pad, + void* userdata) { + TRACKRENDERER_ENTER; DecodedVideoPacket packet; - packet.pts = GST_TIME_AS_MSECONDS(GST_BUFFER_PTS(buffer)); // ns -> ms - packet.surface_data = tbm_surf; - packet.scaler_index = reinterpret_cast(plane_num); - - trackrenderer->eventlistener_->OnMediaPacketVideoDecoded(packet); - return; -} - -void TrackRenderer::GstDecodedVideoCopyBufferCb_(GstElement* element, - GstBuffer* buffer, GstPad* pad, - void* userdata) { - auto caps = gstguard::make_guard(gst_pad_get_current_caps(pad)); - if (!caps.get()) { - TRACKRENDERER_ERROR("caps is NULL"); - return; - } - GstVideoInfo info; - if (!gst_video_info_from_caps(&info, caps.get())) { - TRACKRENDERER_ERROR("fail to get gst_video_info_from_caps()"); - return; - } - - internal::VideoStreamDataType stream; - //memset(&stream, 0x0, sizeof(stream)); - stream.format = TBM_FORMAT_NV12; - stream.width = info.width; - stream.height = info.height; - stream.elevation[GST_VIDEO_COMP_Y] = info.height; - stream.elevation[GST_VIDEO_COMP_U] = info.height / 2; - auto trackrenderer = static_cast(userdata); + GstMemory* mem; + tbm_surface_h tbm_surf; + tbm_surface_info_s info; - tbm_surface_h tbm_surf_t = - tbm_surface_create(stream.width, stream.height, TBM_FORMAT_NV12); - - BOOST_SCOPE_EXIT(&tbm_surf_t) { - if (tbm_surf_t) { - tbm_surface_destroy(tbm_surf_t); - tbm_surf_t = nullptr; - } - } - BOOST_SCOPE_EXIT_END + mem = gst_buffer_peek_memory(buffer, 0); + memset(&info, 0, sizeof(info)); - stream.bo[GST_VIDEO_COMP_Y] = - tbm_surface_internal_get_bo(tbm_surf_t, GST_VIDEO_COMP_Y); - if (!stream.bo[GST_VIDEO_COMP_Y]) { - TRACKRENDERER_ERROR("[bo Y] tbm_surface_internal_get_bo failed"); - return; + if (gst_is_tizen_memory(mem)) { + tbm_surf = (tbm_surface_h)gst_tizen_memory_get_surface(mem); + packet.buffer_addr = gst_buffer_ref(buffer); + } else { + tbm_surf = internal::CreateTbmSurfaceWithBuffer(mem, pad); } - stream.bo[GST_VIDEO_COMP_U] = - tbm_surface_internal_get_bo(tbm_surf_t, GST_VIDEO_COMP_U); - if (!stream.bo[GST_VIDEO_COMP_U]) { - TRACKRENDERER_ERROR("[bo C] tbm_surface_internal_get_bo failed"); + if (!tbm_surf) { + TRACKRENDERER_ERROR("failed to get tbm surface"); return; } - tbm_surface_internal_get_plane_data( - tbm_surf_t, GST_VIDEO_COMP_Y, NULL, NULL, - (uint32_t*)&stream.stride[GST_VIDEO_COMP_Y]); - tbm_surface_internal_get_plane_data( - tbm_surf_t, GST_VIDEO_COMP_U, NULL, NULL, - (uint32_t*)&stream.stride[GST_VIDEO_COMP_U]); - - if (GST_VIDEO_INFO_FORMAT(&info) == GST_VIDEO_FORMAT_STV0) { - GstStructure* s = GST_STRUCTURE_CAST(gst_mini_object_get_qdata( - GST_MINI_OBJECT(buffer), g_quark_from_static_string("v4l2_info"))); - if (!s) { - TRACKRENDERER_ERROR("scaler buffer don't have v4l2_info structure"); - return; - } - - int y_linesize, c_linesize, plane_num = 0; - unsigned int y_viraddr_i, c_viraddr_i = 0; - uintptr_t y_viraddr, c_viraddr; - gst_structure_get_int(s, "y_linesize", &y_linesize); - gst_structure_get_int(s, "u_linesize", &c_linesize); - gst_structure_get_int(s, "plane_num", &plane_num); - gst_structure_get_uint(s, "y_viraddr", &y_viraddr_i); - gst_structure_get_uint(s, "u_viraddr", &c_viraddr_i); - - y_viraddr = (uintptr_t)y_viraddr_i; - c_viraddr = (uintptr_t)c_viraddr_i; - if (!internal::CopyHwCodec(stream, y_viraddr, c_viraddr, y_linesize, - c_linesize)) { - TRACKRENDERER_ERROR("data copy fail"); - return; - } - } else if (GST_VIDEO_INFO_FORMAT(&info) == GST_VIDEO_FORMAT_STV1) { - if (!internal::CopySwCodec(buffer, stream)) { - TRACKRENDERER_ERROR("data copy fail"); - return; - } - } - - tbm_surface_h tbm_surf = internal::CreateTbmSurfaceWithBos(stream); +#ifdef __DEBUG__ + internal::DumpVideoFrame(tbm_surf); +#endif - DecodedVideoPacket packet; packet.pts = GST_TIME_AS_MSECONDS(GST_BUFFER_PTS(buffer)); // ns -> ms packet.surface_data = tbm_surf; trackrenderer->eventlistener_->OnMediaPacketVideoDecoded(packet); + + TRACKRENDERER_LEAVE; return; } diff --git a/src/trackrenderer_capi_utils.cpp b/src/trackrenderer_capi_utils.cpp index a899c23..1c2974e 100644 --- a/src/trackrenderer_capi_utils.cpp +++ b/src/trackrenderer_capi_utils.cpp @@ -721,7 +721,7 @@ TrackRendererDecodedVideoPacket ConvertToDecodedVideoPacket( _packet.pts = packet.pts; _packet.duration = packet.duration; _packet.surface_data = static_cast(packet.surface_data); - _packet.scaler_index = packet.scaler_index; + _packet.buffer_addr = packet.buffer_addr; return _packet; } DecodedVideoFrameBufferType ConvertToVideoFrameBufferType( -- 2.34.1