From 48a29088fe5bb26d0ea94769bdcebd8c0d98fcde Mon Sep 17 00:00:00 2001 From: Suhaspoornachandra Date: Mon, 6 Mar 2023 15:06:54 +0530 Subject: [PATCH] [M108 Migration] Support new pixel format in webrtc Support PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER and PIXEL_FORMAT_ENCODED. Reference: https://review.tizen.org/gerrit/#/c/279885 Change-Id: I30659a3c071b697ed06ff8eaf844ca82799d2b55 Signed-off-by: Suhaspoornachandra --- .../public/remoting_proto_enum_utils.cc | 3 + media/base/video_frame.cc | 92 ++++++++++++++++++++++ media/base/video_frame.h | 16 ++++ media/base/video_frame_layout.cc | 6 ++ media/base/video_types.cc | 19 +++++ media/base/video_types.h | 13 ++- media/mojo/mojom/stable/BUILD.gn | 9 +++ .../mojom/stable/stable_video_decoder_types.mojom | 7 ++ .../stable_video_decoder_types_mojom_traits.h | 26 ++++++ media/video/gpu_memory_buffer_video_frame_pool.cc | 3 + .../openscreen/src/cast/streaming/remoting.proto | 1 + 11 files changed, 192 insertions(+), 3 deletions(-) diff --git a/components/cast_streaming/public/remoting_proto_enum_utils.cc b/components/cast_streaming/public/remoting_proto_enum_utils.cc index 98af814..79b80df 100644 --- a/components/cast_streaming/public/remoting_proto_enum_utils.cc +++ b/components/cast_streaming/public/remoting_proto_enum_utils.cc @@ -374,6 +374,9 @@ absl::optional ToMediaVideoPixelFormat( CASE_RETURN_OTHER(PIXEL_FORMAT_P016LE); CASE_RETURN_OTHER(PIXEL_FORMAT_XR30); CASE_RETURN_OTHER(PIXEL_FORMAT_XB30); +#if BUILDFLAG(IS_TIZEN) + CASE_RETURN_OTHER(PIXEL_FORMAT_ENCODED); +#endif #if defined(TIZEN_TBM_SUPPORT) CASE_RETURN_OTHER(PIXEL_FORMAT_TBM_SURFACE); #endif diff --git a/media/base/video_frame.cc b/media/base/video_frame.cc index fd0dd69..d845a3c 100644 --- a/media/base/video_frame.cc +++ b/media/base/video_frame.cc @@ -164,6 +164,9 @@ gfx::Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) { case PIXEL_FORMAT_YUV420P12: case PIXEL_FORMAT_P016LE: case PIXEL_FORMAT_YUV420AP10: +#if defined(TIZEN_TBM_SUPPORT) + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: +#endif return gfx::Size(2, 2); case PIXEL_FORMAT_UYVY: @@ -179,6 +182,9 @@ gfx::Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) { case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_RGBAF16: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif #if defined(TIZEN_TBM_SUPPORT) case PIXEL_FORMAT_TBM_SURFACE: #endif @@ -215,8 +221,12 @@ static bool RequiresEvenSizeAllocation(VideoPixelFormat format) { case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_RGBAF16: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif #if defined(TIZEN_TBM_SUPPORT) case PIXEL_FORMAT_TBM_SURFACE: + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: #endif return false; case PIXEL_FORMAT_NV12: @@ -400,6 +410,38 @@ scoped_refptr VideoFrame::WrapTBMSurface( frame->buffer_handle_ = handle; return frame; } + +#if BUILDFLAG(IS_TIZEN_TV) +scoped_refptr VideoFrame::WrapTBMInterProcessBuffer( + const gfx::Size& size, + base::TimeDelta timestamp, + gfx::TbmBufferHandle handle) { + const VideoPixelFormat format = PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER; +#if defined(TIZEN_VIDEO_HOLE) + const StorageType storage = STORAGE_HOLE; +#else + const StorageType storage = STORAGE_UNKNOWN; +#endif + const gfx::Rect visible_rect = gfx::Rect(size); + if (!IsValidConfig(format, storage, size, visible_rect, size)) { + DLOG(ERROR) << __FUNCTION__ << " WrapTBMInterProcessBuffer Invalid config." + << ConfigToString(format, storage, size, visible_rect, size); + return nullptr; + } + + auto layout = VideoFrameLayout::CreateWithStrides( + format, size, {handle.strides[0], handle.strides[1]}); + if (!layout) { + DLOG(ERROR) << "Invalid layout."; + return nullptr; + } + + scoped_refptr frame( + new VideoFrame(*layout, storage, gfx::Rect(size), size, timestamp)); + frame->buffer_handle_ = handle; + return frame; +} +#endif #endif // static @@ -960,6 +1002,13 @@ scoped_refptr VideoFrame::WrapVideoFrame( frame = frame->wrapped_frame_; } +#if defined(TIZEN_TBM_SUPPORT) + if (frame->format() == PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER || + frame->format() == PIXEL_FORMAT_TBM_SURFACE) { + wrapping_frame->buffer_handle_ = frame->buffer_handle_; + } +#endif + wrapping_frame->wrapped_frame_ = std::move(frame); return wrapping_frame; } @@ -1111,6 +1160,9 @@ int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) { case PIXEL_FORMAT_XBGR: case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif return 4; case PIXEL_FORMAT_RGB24: return 3; @@ -1130,6 +1182,9 @@ int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) { case PIXEL_FORMAT_YUV422AP10: case PIXEL_FORMAT_YUV444AP10: return 2; +#if defined(TIZEN_TBM_SUPPORT) + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: +#endif case PIXEL_FORMAT_NV12: case PIXEL_FORMAT_NV21: { static const int bytes_per_element[] = {1, 2}; @@ -1321,6 +1376,29 @@ T VideoFrame::GetVisibleDataInternal(T data, size_t plane) const { const gfx::Size subsample = SampleSize(format(), plane); DCHECK(offset.x() % subsample.width() == 0); DCHECK(offset.y() % subsample.height() == 0); + +#if defined(TIZEN_TBM_SUPPORT) + const VideoPixelFormat fmt = format(); + if (PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER == fmt) { + if (!bufmgr_) + bufmgr_ = tbm_bufmgr_init(-1); + { + base::AutoLock autolock(tbm_map_lock_); + if (!vp_[plane]) { + bo_[plane] = tbm_bo_import(bufmgr_, buffer_handle_.key[plane]); + bo_handle_[plane] = + tbm_bo_map(bo_[plane], TBM_DEVICE_CPU, TBM_OPTION_READ); + + vp_[plane] = + (uint8_t*)bo_handle_[plane].ptr + + stride(plane) * (offset.y() / subsample.height()) + // Row offset. + BytesPerElement(format(), plane) * // Column offset. + (offset.x() / subsample.width()); + } + } + return vp_[plane]; + } +#endif return data + stride(plane) * (offset.y() / subsample.height()) + // Row offset. BytesPerElement(format(), plane) * // Column offset. @@ -1480,6 +1558,9 @@ VideoFrame::VideoFrame(const VideoFrameLayout& layout, << coded_size().ToString(); memset(&mailbox_holders_, 0, sizeof(mailbox_holders_)); memset(&data_, 0, sizeof(data_)); +#if defined(TIZEN_TBM_SUPPORT) + memset(bo_handle_, 0, sizeof(tbm_bo_handle) * TBM_BO_NUM_MAX); +#endif } VideoFrame::~VideoFrame() { @@ -1497,6 +1578,17 @@ VideoFrame::~VideoFrame() { for (auto& callback : done_callbacks_) std::move(callback).Run(); + +#if defined(TIZEN_TBM_SUPPORT) + for (int i = 0; i < TBM_BO_NUM_MAX; i++) { + if (bo_handle_[i].ptr) { + if (!tbm_bo_unmap(bo_[i])) + LOG(WARNING) << "unmap bo failed!"; + tbm_bo_unref(bo_[i]); + bo_handle_[i].ptr = nullptr; + } + } +#endif } // static diff --git a/media/base/video_frame.h b/media/base/video_frame.h index b856464..224802f 100644 --- a/media/base/video_frame.h +++ b/media/base/video_frame.h @@ -67,7 +67,9 @@ class GLES2Interface; namespace media { +#if defined(TIZEN_TBM_SUPPORT) constexpr auto TBM_BO_NUM_MAX = 4; +#endif class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe { public: @@ -357,6 +359,12 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe { const gfx::Size& natural_size); #if defined(TIZEN_TBM_SUPPORT) +#if BUILDFLAG(IS_TIZEN_TV) + static scoped_refptr WrapTBMInterProcessBuffer( + const gfx::Size& size, + base::TimeDelta timestamp, + gfx::TbmBufferHandle handle); +#endif // Needed when we have video-frame content in tbm surface. static scoped_refptr WrapTBMSurface(const gfx::Size& size, base::TimeDelta timestamp, @@ -834,6 +842,14 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe { gpu::gles2::GLES2Interface* gl_; unsigned image_id_; scoped_refptr context_provider_; + + mutable tbm_bo bo_[TBM_BO_NUM_MAX] = {nullptr}; + mutable tbm_bo_handle bo_handle_[TBM_BO_NUM_MAX] = { + {nullptr}, + }; + mutable tbm_key key_[TBM_BO_NUM_MAX] = {0}; + mutable uint8_t* vp_[TBM_BO_NUM_MAX] = {nullptr}; + mutable tbm_bufmgr bufmgr_ = {nullptr}; #endif base::TimeDelta timestamp_; diff --git a/media/base/video_frame_layout.cc b/media/base/video_frame_layout.cc index 47bbbffa..760f945 100644 --- a/media/base/video_frame_layout.cc +++ b/media/base/video_frame_layout.cc @@ -56,10 +56,16 @@ size_t VideoFrameLayout::NumPlanes(VideoPixelFormat format) { case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_RGBAF16: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif return 1; case PIXEL_FORMAT_NV12: case PIXEL_FORMAT_NV21: case PIXEL_FORMAT_P016LE: +#if defined(TIZEN_TBM_SUPPORT) + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: +#endif return 2; case PIXEL_FORMAT_I420: case PIXEL_FORMAT_YV12: diff --git a/media/base/video_types.cc b/media/base/video_types.cc index 4edcc27..f506483 100644 --- a/media/base/video_types.cc +++ b/media/base/video_types.cc @@ -85,9 +85,15 @@ std::string VideoPixelFormatToString(VideoPixelFormat format) { return "PIXEL_FORMAT_YUV422AP10"; case PIXEL_FORMAT_YUV444AP10: return "PIXEL_FORMAT_YUV444AP10"; +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: + return "PIXEL_FORMAT_ENCODED"; +#endif #if defined(TIZEN_TBM_SUPPORT) case PIXEL_FORMAT_TBM_SURFACE: return "PIXEL_FORMAT_TBM_SURFACE"; + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: + return "PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER"; #endif } NOTREACHED() << "Invalid VideoPixelFormat provided: " << format; @@ -149,6 +155,9 @@ bool IsYuvPlanar(VideoPixelFormat format) { case PIXEL_FORMAT_YUV420AP10: case PIXEL_FORMAT_YUV422AP10: case PIXEL_FORMAT_YUV444AP10: +#if defined(TIZEN_TBM_SUPPORT) + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: +#endif return true; case PIXEL_FORMAT_UNKNOWN: @@ -165,6 +174,9 @@ bool IsYuvPlanar(VideoPixelFormat format) { case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_RGBAF16: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif #if defined(TIZEN_TBM_SUPPORT) case PIXEL_FORMAT_TBM_SURFACE: #endif @@ -248,6 +260,7 @@ bool IsOpaque(VideoPixelFormat format) { case PIXEL_FORMAT_XB30: #if defined(TIZEN_TBM_SUPPORT) case PIXEL_FORMAT_TBM_SURFACE: + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: #endif return true; case PIXEL_FORMAT_I420A: @@ -260,6 +273,9 @@ bool IsOpaque(VideoPixelFormat format) { case PIXEL_FORMAT_YUV420AP10: case PIXEL_FORMAT_YUV422AP10: case PIXEL_FORMAT_YUV444AP10: +#if BUILDFLAG(IS_TIZEN) + case PIXEL_FORMAT_ENCODED: +#endif break; } return false; @@ -291,6 +307,9 @@ size_t BitDepth(VideoPixelFormat format) { case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_I422A: case PIXEL_FORMAT_I444A: +#if defined(TIZEN_TBM_SUPPORT) + case PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER: +#endif return 8; case PIXEL_FORMAT_YUV420P9: case PIXEL_FORMAT_YUV422P9: diff --git a/media/base/video_types.h b/media/base/video_types.h index d7dad6d..c6b3709 100644 --- a/media/base/video_types.h +++ b/media/base/video_types.h @@ -89,13 +89,20 @@ enum VideoPixelFormat { PIXEL_FORMAT_YUV444AP10 = 38, // Please update UMA histogram enumeration when adding new formats here. +#if BUILDFLAG(IS_TIZEN) + PIXEL_FORMAT_ENCODED = 39, // encoded pixels, 1 plane #if defined(TIZEN_TBM_SUPPORT) - PIXEL_FORMAT_TBM_SURFACE = 39, - PIXEL_FORMAT_MAX = PIXEL_FORMAT_TBM_SURFACE, + PIXEL_FORMAT_TBM_SURFACE = 40, + PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER = 41, + PIXEL_FORMAT_MAX = PIXEL_FORMAT_TBM_INTER_PROCESS_BUFFER, +#else + PIXEL_FORMAT_MAX = + PIXEL_FORMAT_ENCODED, // Must always be equal to largest entry logged. +#endif // defined(TIZEN_TBM_SUPPORT) #else PIXEL_FORMAT_MAX = PIXEL_FORMAT_YUV444AP10, // Must always be equal to largest entry logged. -#endif +#endif // defined(OS_TIZEN) }; // These values are persisted to logs. Entries should not be renumbered and diff --git a/media/mojo/mojom/stable/BUILD.gn b/media/mojo/mojom/stable/BUILD.gn index 205688b..2cfd7dd 100644 --- a/media/mojo/mojom/stable/BUILD.gn +++ b/media/mojo/mojom/stable/BUILD.gn @@ -4,6 +4,9 @@ import("//media/gpu/args.gni") import("//mojo/public/tools/bindings/mojom.gni") +if (use_efl) { + import("//tizen_src/build/config/tizen_features.gni") +} mojom("stable_video_decoder") { generate_java = true @@ -17,6 +20,12 @@ mojom("stable_video_decoder") { if (is_linux || is_chromeos_ash) { enabled_features += [ "is_linux_or_chromeos_ash" ] } + if (is_tizen) { + enabled_features += [ "is_tizen" ] + } + if (tizen_tbm_support) { + enabled_features += [ "tizen_tbm_support" ] + } public_deps = [ ":native_pixmap_handle", diff --git a/media/mojo/mojom/stable/stable_video_decoder_types.mojom b/media/mojo/mojom/stable/stable_video_decoder_types.mojom index 02acd4e..637b44e 100644 --- a/media/mojo/mojom/stable/stable_video_decoder_types.mojom +++ b/media/mojo/mojom/stable/stable_video_decoder_types.mojom @@ -400,6 +400,13 @@ enum VideoPixelFormat { [MinVersion=1] kPixelFormatYUV420AP10 = 36, [MinVersion=1] kPixelFormatYUV422AP10 = 37, [MinVersion=1] kPixelFormatYUV444AP10 = 38, + + [EnableIf=is_tizen] kPixelFormatEncoded = 39, + + [EnableIf=tizen_tbm_support] kPixelFormatTbmSurface = 40, + + [EnableIf=tizen_tbm_support] kPixelFormatTbmInterProcessBuffer=41, + }; // Based on |media.mojom.EosVideoFrameData|. diff --git a/media/mojo/mojom/stable/stable_video_decoder_types_mojom_traits.h b/media/mojo/mojom/stable/stable_video_decoder_types_mojom_traits.h index 358f5c2..dbeac05 100644 --- a/media/mojo/mojom/stable/stable_video_decoder_types_mojom_traits.h +++ b/media/mojo/mojom/stable/stable_video_decoder_types_mojom_traits.h @@ -1313,6 +1313,17 @@ struct EnumTraits