From: Eunhye Choi Date: Thu, 17 Aug 2023 10:54:11 +0000 (+0900) Subject: Support video capture during playback X-Git-Tag: accepted/tizen/7.0/unified/20230921.172225~1 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=85836115d440ae63f9e52e94f7dbeda2c3c67dce;p=platform%2Fcore%2Fmultimedia%2Fesplusplayer.git Support video capture during playback [Version] 0.0.14 [Issue Type] Add features - Add esplusplayer_get_decoded_video_packet - Add ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_MANUAL_COPY at esplusplayer_decoded_video_frame_buffer_type - the interfaces are synchronized with VD Change-Id: I23b82e3a279abd05b646fd238134b84e6b578092 --- diff --git a/include/esplusplayer_capi/buffer.h b/include/esplusplayer_capi/buffer.h index acaf120..4258e3c 100644 --- a/include/esplusplayer_capi/buffer.h +++ b/include/esplusplayer_capi/buffer.h @@ -51,7 +51,8 @@ typedef enum { ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_NONE, ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_COPY, ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_REFERENCE, - ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_SCALE + ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_SCALE, + ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_MANUAL_COPY } esplusplayer_decoded_video_frame_buffer_type; /** diff --git a/include/esplusplayer_capi/esplusplayer_capi.h b/include/esplusplayer_capi/esplusplayer_capi.h index aa20820..9c17146 100644 --- a/include/esplusplayer_capi/esplusplayer_capi.h +++ b/include/esplusplayer_capi/esplusplayer_capi.h @@ -289,6 +289,27 @@ typedef enum { } esplusplayer_rsc_alloc_policy; /** + * @brief Enumerations for the status of getting decoded video frame + * @version 4.0 + */ +typedef enum { + /** @brief successfully decoded video frame acquired. */ + ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_SUCCESS, + /** @brief it means app has to return the video before getting decoded + * video frame frame. + */ + ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_REMAINING_BUFFER, + /** + * @brief there is no filled video frame yet. + */ + ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_FILLED_BUFFER, + /** + * @brief internal error + */ + ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_UNKNOWN +} esplusplayer_get_decoded_video_frame_status_type; + +/** * @brief Create a esplusplayer handle. * @param None * @return return esplusplayer handle pointer. @@ -1936,6 +1957,65 @@ int esplusplayer_get_render_time_offset(esplusplayer_handle handle, esplusplayer_stream_type type, int64_t* offset); /** + * @brief Requests decoded video frame packet to acquire it. it works only + * with #ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_MANUAL_COPY + * mode + * @param [in] handle : esplusplayer handle. + * @param [out] packet : the decoded buffer. + * @param [out] status : (nullable) the result of video frame requested + * @pre The player state must be one of #ESPLUSPLAYER_STATE_READY or + * #ESPLUSPLAYER_STATE_PAUSED or #ESPLUSPLAYER_STATE_PLAYING. + * @post None + * @return #ESPLUSPLAYER_ERROR_TYPE_NONE on success, otherwise one of + * esplusplayer_error_type values will be returned. + * @exception None + * @version 4.0 + * @see esplusplayer_set_video_frame_buffer_type() + * @see esplusplayer_decoded_buffer_destroy() + * @see esplusplayer_decoded_video_frame_buffer_type + * @code + * ... + * esplusplayer_set_video_frame_buffer_type(handle, + * ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_MANUAL_COPY); + * ... + * esplusplayer_prepare_async(handle); + * ... + * // after prepared + * esplusplayer_decoded_video_packet packet; + * esplusplayer_get_decoded_video_frame_status_type state; + * int retval = esplusplayer_get_decoded_video_packet(handle, &packet, &state); + * if (state == ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_SUCCESS) { + * // successful case. + * // in this case, retval will be ESPLUSPLAYER_ERROR_TYPE_NONE + * // you have to call esplusplayer_decoded_buffer_destroy() after using the + * // packet + * ... + * esplusplayer_decoded_buffer_destroy(handle, &packet); + * } else if (state == + * ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_REMAINING_BUFFER) { + * // app has to call esplusplayer_decoded_buffer_destroy() with previous + * // video packet. + * // it means player couldn't draw the video frame into a buffer due to no + * buffer. + * // in this case ,retval will be ESPLUSPLAYER_ERROR_TYPE_NONE + * } else if (state == + * ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_FILLED_BUFFER) { + * // it means app should retry to get decoded video packet. + * // in most case, there were no buffers drawn in the timing you called this + * // api. + * // in this case, retval will be ESPLUSPLAYER_ERROR_TYPE_NONE + * } else if (state == ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_UNKNOWN) { + * // internal error happened + * // in this case, retval will be ESPLUSPLAYER_ERROR_TYPE_INVALID_OPERATION + * } + * ... + * @endcode + */ +int esplusplayer_get_decoded_video_packet( + esplusplayer_handle handle, esplusplayer_decoded_video_packet* packet, + esplusplayer_get_decoded_video_frame_status_type* status); + +/** * @brief Provided api for enabling video hole. * @param [in] handle : esplusplayer handle. * @param [in] value : the value of video hole. diff --git a/include/plusplayer/esplusplayer.h b/include/plusplayer/esplusplayer.h index df5cdfb..9f283f1 100644 --- a/include/plusplayer/esplusplayer.h +++ b/include/plusplayer/esplusplayer.h @@ -724,6 +724,23 @@ class EsPlusPlayer : private boost::noncopyable { return false; } /** + * @brief Get the decoded video packet. + * @param [out] packet + * @return GetDecodedVideoFrameStatus + */ + virtual GetDecodedVideoFrameStatus GetDecodedPacket( + DecodedVideoPacket& packet) { + return GetDecodedVideoFrameStatus::kUnknown; + } + /** + * @brief Return the decoded video packet. + * @param [in] packet + * @return @c True on success, otherwise @c False + */ + virtual bool ReturnDecodedPacket(const DecodedVideoPacket& packet) { + return false; + } + /** * @brief Provided api for enabling video hole * @return @c True on success, otherwise @c False */ diff --git a/include/plusplayer/types/buffer.h b/include/plusplayer/types/buffer.h index 94b1785..76ca1e8 100644 --- a/include/plusplayer/types/buffer.h +++ b/include/plusplayer/types/buffer.h @@ -56,6 +56,21 @@ struct DecodedVideoPacket { void* buffer_addr = nullptr; }; +struct DecoderBufferTime { + uint64_t pts = 0; + uint64_t system_time = 0; +}; + +/** + * @brief Enumerations for the state of getting decoded packet + */ +enum class GetDecodedVideoFrameStatus { + kSuccess, + kNoRemainingBuffer, + kNoFilledBuffer, + kUnknown, +}; + } // namespace plusplayer #endif // __PLUSPLAYER_TYPES_BUFFER_H__ diff --git a/packaging/esplusplayer.spec b/packaging/esplusplayer.spec index 373cfcd..8eab12a 100644 --- a/packaging/esplusplayer.spec +++ b/packaging/esplusplayer.spec @@ -2,7 +2,7 @@ %bcond_without ESPLUSPLAYER_UT Name: esplusplayer Summary: new multimedia streaming player -Version: 0.0.13 +Version: 0.0.14 Release: 0 Group: Multimedia/Libraries License: Apache-2.0 diff --git a/src/esplusplayer/include_internal/esplayer/decoded_pkt_list.h b/src/esplusplayer/include_internal/esplayer/decoded_pkt_list.h index 59ae836..fd4614e 100644 --- a/src/esplusplayer/include_internal/esplayer/decoded_pkt_list.h +++ b/src/esplusplayer/include_internal/esplayer/decoded_pkt_list.h @@ -136,6 +136,30 @@ class DecodedCopiedPacketList : public AbstractDecodedPacketList { } }; +class ManualDecodedCopiedPacketList : public DecodedPacketManagerInterface { + public: + using Handler = std::function; + explicit ManualDecodedCopiedPacketList(Handler handler) : handler_(handler) { + LOG_DEBUG("created"); + } + virtual ~ManualDecodedCopiedPacketList() { LOG_DEBUG("destroyed"); } + + protected: + virtual bool TryToAdd(esplusplayer_decoded_video_packet* packet) { + return false; + } + virtual void Clear() {} + virtual void GetFreeTbmSurface(void** ptr, bool is_scale_change) {} + + virtual void Remove(esplusplayer_decoded_video_packet* packet) { + if (handler_(packet) == false) { + LOG_ERROR("packet return failed"); + } + } + + private: + Handler handler_; +}; class DecodedScaledPacketList : public AbstractDecodedPacketList { public: explicit DecodedScaledPacketList() { LOG_DEBUG("created"); } diff --git a/src/esplusplayer/include_internal/esplayer/esplayer.h b/src/esplusplayer/include_internal/esplayer/esplayer.h index 1990ade..5bf2247 100644 --- a/src/esplusplayer/include_internal/esplayer/esplayer.h +++ b/src/esplusplayer/include_internal/esplayer/esplayer.h @@ -97,6 +97,9 @@ class EsPlayer : public EsPlusPlayer { bool SetVideoCodecType(const PlayerVideoCodecType& type) override; bool SetRenderTimeOffset(const StreamType type, int64_t offset) override; bool GetRenderTimeOffset(const StreamType type, int64_t* offset) override; + GetDecodedVideoFrameStatus GetDecodedPacket( + DecodedVideoPacket& packet) override; + bool ReturnDecodedPacket(const DecodedVideoPacket& packet) override; bool EnableVideoHole(bool value) override; private: diff --git a/src/esplusplayer/src/esplayer.cpp b/src/esplusplayer/src/esplayer.cpp index 509688b..1e12436 100644 --- a/src/esplusplayer/src/esplayer.cpp +++ b/src/esplusplayer/src/esplayer.cpp @@ -856,6 +856,24 @@ bool EsPlayer::SetStream(const VideoStreamPtr& stream) { return SetStream_(track); } +GetDecodedVideoFrameStatus EsPlayer::GetDecodedPacket( + DecodedVideoPacket& packet) { + if (state_manager_.GetState() < EsState::kReady) { + LOG_ERROR_P(this, "Invalid State , current %d", + state_manager_.GetStateEnum()); + return GetDecodedVideoFrameStatus::kUnknown; + } + return trackrenderer_->GetDecodedPacket(packet); +} + +bool EsPlayer::ReturnDecodedPacket(const DecodedVideoPacket& packet) { + if (state_manager_.GetState() < EsState::kReady) { + LOG_ERROR_P(this, "Invalid State , current %d", + state_manager_.GetStateEnum()); + return false; + } + return trackrenderer_->ReturnDecodedPacket(packet); +} void EsPlayer::ResetContextForClose_() { internal::ResetDrmProperty(drm_property_); diff --git a/src/esplusplayer/src/esplusplayer_capi.cpp b/src/esplusplayer/src/esplusplayer_capi.cpp index 1b95cb9..0cbcb24 100644 --- a/src/esplusplayer/src/esplusplayer_capi.cpp +++ b/src/esplusplayer/src/esplusplayer_capi.cpp @@ -653,6 +653,27 @@ inline int convert_return_type_(bool ret) { : ESPLUSPLAYER_ERROR_TYPE_INVALID_OPERATION; } +inline esplusplayer_get_decoded_video_frame_status_type +convert_get_decoded_video_frame_status_( + const plusplayer::GetDecodedVideoFrameStatus status) { + switch (status) { + case plusplayer::GetDecodedVideoFrameStatus::kSuccess: { + return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_SUCCESS; + } + case plusplayer::GetDecodedVideoFrameStatus::kNoRemainingBuffer: { + return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_REMAINING_BUFFER; + } + case plusplayer::GetDecodedVideoFrameStatus::kNoFilledBuffer: { + return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_NO_FILLED_BUFFER; + } + case plusplayer::GetDecodedVideoFrameStatus::kUnknown: { + return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_UNKNOWN; + } + default: { return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_UNKNOWN; } + } + return ESPLUSPLAYER_GET_DECVIDEOFRAME_STATUS_UNKNOWN; +} + inline VideoStreamPtr convert_stream_ptr_( esplusplayer_video_stream_info* from) { LOG_INFO("mime type : %u", static_cast(from->mime_type)); @@ -1019,6 +1040,7 @@ int esplusplayer_get_playing_time(esplusplayer_handle handle, uint64_t* ms) { namespace { std::shared_ptr CreateDecodedPacketManager( + esplusplayer_handle handle, esplusplayer_decoded_video_frame_buffer_type type) { std::shared_ptr mgr = nullptr; if (type == ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_COPY) @@ -1027,6 +1049,16 @@ std::shared_ptr CreateDecodedPacketManager( mgr = std::make_shared(); else if (type == ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_SCALE) mgr = std::make_shared(); + else if (type == ESPLUSPLAYER_DECODED_VIDEO_FRAME_BUFFER_TYPE_MANUAL_COPY) + mgr = std::make_shared( + [handle](esplusplayer_decoded_video_packet* pkt) { + plusplayer::DecodedVideoPacket _pkt; + _pkt.pts = pkt->pts; + _pkt.duration = pkt->duration; + _pkt.surface_data = static_cast(pkt->surface_data); + _pkt.buffer_addr = pkt->private_data; + return cast_(handle)->ReturnDecodedPacket(_pkt); + }); return mgr; } } // namespace @@ -1039,7 +1071,7 @@ int esplusplayer_set_video_frame_buffer_type( return ESPLUSPLAYER_ERROR_TYPE_INVALID_PARAMETER; auto priv = static_cast(handle); - priv->decoded_pkt_mgr = ::CreateDecodedPacketManager(type); + priv->decoded_pkt_mgr = ::CreateDecodedPacketManager(handle, type); priv->listener->SetDecodedPacketManager(priv->decoded_pkt_mgr); auto ret = cast_(handle)->SetVideoFrameBufferType( @@ -1332,6 +1364,31 @@ int esplusplayer_set_first_video_decoding_done_cb( return convert_return_type_(true); } +int esplusplayer_get_decoded_video_packet( + esplusplayer_handle handle, esplusplayer_decoded_video_packet* packet, + esplusplayer_get_decoded_video_frame_status_type* state) { + if (is_null_(handle) || is_null_(packet)) { + LOG_ERROR("handle[%p] or packet[%p] is nil.", handle, packet); + return ESPLUSPLAYER_ERROR_TYPE_INVALID_PARAMETER; + } + plusplayer::DecodedVideoPacket _packet; + bool ret = false; + auto _state = cast_(handle)->GetDecodedPacket(_packet); + if (_state != plusplayer::GetDecodedVideoFrameStatus::kUnknown) { + ret = true; + } + if (_state == plusplayer::GetDecodedVideoFrameStatus::kSuccess) { + packet->pts = _packet.pts; + packet->duration = _packet.duration; + packet->surface_data = static_cast(_packet.surface_data); + packet->private_data = _packet.buffer_addr; + } + if (state) { + *state = convert_get_decoded_video_frame_status_(_state); + } + return convert_return_type_(ret); +} + int esplusplayer_decoded_buffer_destroy( esplusplayer_handle handle, esplusplayer_decoded_video_packet* packet) { if (is_null_(handle)) { diff --git a/src/plusplayer-core/include_internal/core/trackrendereradapter.h b/src/plusplayer-core/include_internal/core/trackrendereradapter.h index 7f7aa54..72bca4f 100644 --- a/src/plusplayer-core/include_internal/core/trackrendereradapter.h +++ b/src/plusplayer-core/include_internal/core/trackrendereradapter.h @@ -140,6 +140,8 @@ class TrackRendererAdapter { bool Flush(const StreamType& type); bool Flush(const TrackType& type); void GetAttribute(const Attribute& attr, boost::any* value); + GetDecodedVideoFrameStatus GetDecodedPacket(DecodedVideoPacket& packet); + bool ReturnDecodedPacket(const DecodedVideoPacket& packet); bool EnableVideoHole(bool value); private: diff --git a/src/plusplayer-core/include_internal/core/trackrendereradapter_utils.h b/src/plusplayer-core/include_internal/core/trackrendereradapter_utils.h index da49c74..2276c06 100644 --- a/src/plusplayer-core/include_internal/core/trackrendereradapter_utils.h +++ b/src/plusplayer-core/include_internal/core/trackrendereradapter_utils.h @@ -20,6 +20,7 @@ #include "trackrenderer_capi/error.h" #include "trackrenderer_capi/track.h" #include "trackrenderer_capi/trackrenderer_capi.h" +#include "trackrenderer_capi/trackrenderer_internal.h" namespace plusplayer { @@ -47,8 +48,12 @@ SubtitleType ConvertToSubtitleType(const TrackRendererSubtitleType& type); TrackType ConvertToTrackType(const TrackRendererTrackType typevalue); DecodedVideoPacket ConvertToDecodedVideoPacket( const TrackRendererDecodedVideoPacket* packet); +TrackRendererDecodedVideoPacket ConvertToDecodedVideoPacket( + const DecodedVideoPacket& packet); TrackRendererDecodedVideoFrameBufferType ConvertToVideoFrameBufferType( const DecodedVideoFrameBufferType& type); +GetDecodedVideoFrameStatus ConvertToGetDecodedVideoFrameStatus( + const TrackRendererGetDecodedVideoFrameState state); TrackRendererDisplayMode ConvertToTrackRendererDisplayMode( const DisplayMode& mode); TrackRendererDisplayRotate ConvertToTrackRendererDisplayRotate( diff --git a/src/plusplayer-core/src/trackrendereradapter.cpp b/src/plusplayer-core/src/trackrendereradapter.cpp index 72dcf3f..43b7645 100644 --- a/src/plusplayer-core/src/trackrendereradapter.cpp +++ b/src/plusplayer-core/src/trackrendereradapter.cpp @@ -642,6 +642,24 @@ void TrackRendererAdapter::SetVideoFrameBufferType( handle_, adapter_utils::ConvertToVideoFrameBufferType(type)); } +GetDecodedVideoFrameStatus TrackRendererAdapter::GetDecodedPacket( + DecodedVideoPacket& packet) { + TrackRendererDecodedVideoPacket _packet; + TrackRendererGetDecodedVideoFrameState state; + trackrenderer_get_decoded_video_frame(handle_, &_packet, &state); + packet = adapter_utils::ConvertToDecodedVideoPacket(&_packet); + return adapter_utils::ConvertToGetDecodedVideoFrameStatus(state); +} + +bool TrackRendererAdapter::ReturnDecodedPacket( + const DecodedVideoPacket& packet) { + auto _packet = adapter_utils::ConvertToDecodedVideoPacket(packet); + if (trackrenderer_return_decoded_video_frame(handle_, &_packet) == kFailed) { + return false; + } + return true; +} + bool TrackRendererAdapter::EnableVideoHole(bool value) { return (trackrenderer_enable_video_hole(handle_, value) != kFailed); } diff --git a/src/plusplayer-core/src/trackrendereradapter_utils.cpp b/src/plusplayer-core/src/trackrendereradapter_utils.cpp index 124fdeb..3703fc2 100644 --- a/src/plusplayer-core/src/trackrendereradapter_utils.cpp +++ b/src/plusplayer-core/src/trackrendereradapter_utils.cpp @@ -475,6 +475,16 @@ DecodedVideoPacket ConvertToDecodedVideoPacket( return _packet; } +TrackRendererDecodedVideoPacket ConvertToDecodedVideoPacket( + const DecodedVideoPacket& packet) { + TrackRendererDecodedVideoPacket _packet; + _packet.pts = packet.pts; + _packet.duration = packet.duration; + _packet.surface_data = static_cast(packet.surface_data); + _packet.buffer_addr = packet.buffer_addr; + return _packet; +} + TrackRendererDecodedVideoFrameBufferType ConvertToVideoFrameBufferType( const DecodedVideoFrameBufferType& type) { switch (type) { @@ -496,6 +506,27 @@ TrackRendererDecodedVideoFrameBufferType ConvertToVideoFrameBufferType( } } +GetDecodedVideoFrameStatus ConvertToGetDecodedVideoFrameStatus( + const TrackRendererGetDecodedVideoFrameState state) { + switch (state) { + case TrackRendererGetDecodedVideoFrameStateErrorNone: { + return GetDecodedVideoFrameStatus::kSuccess; + } + case TrackRendererGetDecodedVideoFrameStateNoRemainingBufferError: { + return GetDecodedVideoFrameStatus::kNoRemainingBuffer; + } + case TrackRendererGetDecodedVideoFrameStateNoFilledBufferError: { + return GetDecodedVideoFrameStatus::kNoFilledBuffer; + } + case TrackRendererGetDecodedVideoFrameStateUnknownError: { + return GetDecodedVideoFrameStatus::kUnknown; + } + default: + LOG_ERROR("wrong state type"); + return GetDecodedVideoFrameStatus::kUnknown; + } +} + TrackRendererDisplayMode ConvertToTrackRendererDisplayMode( const DisplayMode& mode) { switch (mode) {