[WebRTC] Optimize shared memory creation for video capture 55/293455/6
authorJakub Gajownik <j.gajownik2@samsung.com>
Tue, 9 May 2023 09:13:25 +0000 (11:13 +0200)
committerm.jurkiewicz <m.jurkiewicz@samsung.com>
Mon, 29 May 2023 13:00:25 +0000 (15:00 +0200)
Before this CL, every time when encoded buffer from
video capture is send to browser process, new shared memory
was created. As there's already shared memory in video capture
buffer pool from which we receive buffer, there is no need
to allocate new buffer and copy data to it.
We'll benefit from eliminating shared memory creation and
data coping to newly created buffer.
To properly release buffer to pool new IPC message is added -
BufferConsumed. It is called as soon as data is consumed
(pushed to platform player) and this data is no longer needed.

Bug: https://cam.sprc.samsung.pl/browse/VDGAME-279
Change-Id: I2fbbd38332b4cd3bf30ddfb7003727bd7c2b9879
Signed-off-by: Jakub Gajownik <j.gajownik2@samsung.com>
15 files changed:
third_party/blink/renderer/platform/video_capture/video_capture_impl.cc
third_party/blink/renderer/platform/video_capture/video_capture_impl.h
tizen_src/chromium_impl/content/browser/media/tizen/browser_media_player_manager_efl.cc
tizen_src/chromium_impl/content/browser/media/tizen/browser_media_player_manager_efl.h
tizen_src/chromium_impl/content/common/media/media_player_messages_efl.h
tizen_src/chromium_impl/content/renderer/media/tizen/renderer_media_player_manager_efl.cc
tizen_src/chromium_impl/content/renderer/media/tizen/renderer_media_player_manager_efl.h
tizen_src/chromium_impl/media/base/tizen/demuxer_stream_player_params_efl.h
tizen_src/chromium_impl/media/base/tizen/media_player_manager_efl.h
tizen_src/chromium_impl/media/base/tizen/media_source_esplus_player_capi.cc
tizen_src/chromium_impl/media/base/tizen/suspending_media_player.cc
tizen_src/chromium_impl/media/base/tizen/suspending_media_player.h
tizen_src/chromium_impl/media/blink/mm_video_decoder.cc
tizen_src/chromium_impl/media/blink/mm_video_decoder.h
tizen_src/chromium_impl/media/test/mock_media_player_manager.h

index ad2033ca00d8a196f933cf9ab0acd1b1f2e5e13e..4d826a3141bc27fff3b4750d1c704f8ef7a7f0a2 100644 (file)
@@ -174,6 +174,12 @@ struct VideoCaptureImpl::BufferContext
     }
   }
 
+#if defined(OS_TIZEN_TV_PRODUCT)
+  base::UnsafeSharedMemoryRegion GetSharedMemoryRegion() {
+    return region_.Duplicate();
+  }
+#endif  // defined(OS_TIZEN_TV_PRODUCT)
+
  private:
   void InitializeFromSharedMemory(mojo::ScopedSharedBufferHandle handle) {
     DCHECK(handle.is_valid());
@@ -183,7 +189,8 @@ struct VideoCaptureImpl::BufferContext
       DLOG(ERROR) << "Unwrapping shared memory failed.";
       return;
     }
-    writable_mapping_ = region.Map();
+    region_ = std::move(region);
+    writable_mapping_ = region_.Map();
     if (!writable_mapping_.IsValid()) {
       DLOG(ERROR) << "Mapping shared memory failed.";
       return;
@@ -232,6 +239,7 @@ struct VideoCaptureImpl::BufferContext
   VideoFrameBufferHandleType buffer_type_;
 
   // Only valid for |buffer_type_ == SHARED_BUFFER_HANDLE|.
+  base::UnsafeSharedMemoryRegion region_;
   base::WritableSharedMemoryMapping writable_mapping_;
 
   // Only valid for |buffer_type_ == READ_ONLY_SHMEM_REGION|.
@@ -927,7 +935,7 @@ void VideoCaptureImpl::OnBufferReady(
                 media::MMVideoDecoder::CanPushFrameResult::kError) {
           buffer->info->metadata.player_id =
               platform_video_decoder_->GetPlayerId();
-          PushBufferToPlatformPlayer(buffer->buffer_id, buffer->info);
+          PushBufferToPlatformPlayer(std::move(buffer));
           // do not deliver encoded frame here. the local platform player will
           // give us tbm frame which could be shared inter-process.
           return;
@@ -1067,30 +1075,31 @@ void VideoCaptureImpl::OnBufferDestroyed(int32_t buffer_id) {
 
 #if defined(OS_TIZEN_TV_PRODUCT)
 void VideoCaptureImpl::PushBufferToPlatformPlayer(
-    int32_t buffer_id,
-    const media::mojom::blink::VideoFrameInfoPtr& info) {
+    media::mojom::blink::ReadyBufferPtr buffer) {
   LOG(INFO) << "VideoCaptureImpl::PushBufferToPlatformPlayer";
-  if (platform_video_decoder_ &&
-      platform_video_decoder_->CanPushFrame() ==
+  if (!platform_video_decoder_ ||
+      platform_video_decoder_->CanPushFrame() !=
           media::MMVideoDecoder::CanPushFrameResult::kOk) {
-    const auto& iter = client_buffers_.find(buffer_id);
-    DCHECK(iter != client_buffers_.end());
-    scoped_refptr<BufferContext> buffer_context = iter->second;
-    platform_video_decoder_->PushFrame(
-        static_cast<const uint8_t*>(buffer_context->data()),
-        info->encoded_data_size,
-        media::MMVideoDecoder::PendingFrame{
-            info->coded_size,
-            static_cast<uint32_t>(info->timestamp.InMilliseconds()), 1,
-            info->metadata.capture_begin_time.value_or(base::TimeTicks::Now()),
-            true, info->metadata});
-  } else {
     LOG(ERROR) << "Can't use hardware to play camera stream.";
     // OnFrameDropped(media::VideoCaptureFrameDropReason::
     //                  kVideoCaptureImplFailedToWrapDataAsMediaVideoFrame);
+    GetVideoCaptureHost()->ReleaseBuffer(device_id_, buffer->buffer_id,
+                                         media::VideoCaptureFeedback());
+    return;
   }
-  GetVideoCaptureHost()->ReleaseBuffer(device_id_, buffer_id,
-                                       media::VideoCaptureFeedback());
+
+  const auto& iter = client_buffers_.find(buffer->buffer_id);
+  DCHECK(iter != client_buffers_.end());
+  scoped_refptr<BufferContext> buffer_context = iter->second;
+  platform_video_decoder_->PushFrame(
+      buffer->buffer_id, buffer_context->GetSharedMemoryRegion(),
+      buffer->info->encoded_data_size,
+      media::MMVideoDecoder::PendingFrame{
+          buffer->info->coded_size,
+          static_cast<uint32_t>(buffer->info->timestamp.InMilliseconds()), 1,
+          buffer->info->metadata.capture_begin_time.value_or(
+              base::TimeTicks::Now()),
+          true, buffer->info->metadata, base::TimeDelta(), buffer->buffer_id});
 }
 
 void VideoCaptureImpl::OnFrameFromPlatformPlayer(
@@ -1106,6 +1115,11 @@ void VideoCaptureImpl::OnFrameFromPlatformPlayer(
     client.second.deliver_frame_cb.Run(frame, scaled_video_frames,
                                        base::TimeTicks::Now());
 }
+
+void VideoCaptureImpl::ReleaseBuffer(int32_t buffer_id) {
+  GetVideoCaptureHost()->ReleaseBuffer(device_id_, buffer_id,
+                                       media::VideoCaptureFeedback());
+}
 #endif
 
 void VideoCaptureImpl::OnSetAiZoomSettings(
@@ -1234,6 +1248,8 @@ void VideoCaptureImpl::StartCaptureInternal() {
       media::BindToCurrentLoop(
           base::BindRepeating(&VideoCaptureImpl::OnFrameFromPlatformPlayer,
                               base::Unretained(this))),
+      media::BindToCurrentLoop(base::BindRepeating(
+          &VideoCaptureImpl::ReleaseBuffer, base::Unretained(this))),
       media::RtcCallType::NONE, params_.ai_zoom_settings);
 #endif
 
index f460ef8aca19279c51451c68a1d04618b4040745..1baffd7167bf37cdf74b176876a3432ed5ed7c85 100644 (file)
@@ -132,11 +132,10 @@ class PLATFORM_EXPORT VideoCaptureImpl
       base::TimeDelta::FromSeconds(10);
 
 #if defined(OS_TIZEN_TV_PRODUCT)
-  void PushBufferToPlatformPlayer(
-      int32_t buffer_id,
-      const media::mojom::blink::VideoFrameInfoPtr& info);
+  void PushBufferToPlatformPlayer(media::mojom::blink::ReadyBufferPtr buffer);
   void OnFrameFromPlatformPlayer(scoped_refptr<media::VideoFrame> frame,
                                  uint32_t timestamp);
+  void ReleaseBuffer(int32_t buffer_id);
 #endif
 
   void OnSetAiZoomSettings(const media::TizenAiZoomSettings& settings);
index c0f0355c85ccc5e8e3e3f87aa7c06922dccce6d7..f7c8cda9a2a0edf7d0bdf03e5fa9a09d29fab0e1 100644 (file)
@@ -439,6 +439,12 @@ bool BrowserMediaPlayerManagerEfl::Send(int player_id, IPC::Message* msg) {
 }
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+void BrowserMediaPlayerManagerEfl::OnBufferConsumed(int player_id,
+                                                    int32_t buffer_id) {
+  Send(player_id, new MediaPlayerEflMsg_BufferConsumed(GetRoutingID(player_id),
+                                                       player_id, buffer_id));
+}
+
 void BrowserMediaPlayerManagerEfl::SetFastMediaSwitchCallback(
     WebContents* web_contents) {
   if (WebContentsDelegateEfl* delegate =
index 9e8a4e900f634e97a22eea72e373460165c6b390..9a6155c783f62b103644a4b057fa6dcfc2fd8267 100644 (file)
@@ -134,6 +134,7 @@ class CONTENT_EXPORT BrowserMediaPlayerManagerEfl
 #endif  // defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+  void OnBufferConsumed(int player_id, int32_t buffer_id) override;
   void OnSetDecryptorHandle(int player_id, eme::eme_decryptor_t decryptor);
   void OnInitData(int player_id,
                   const std::vector<unsigned char>& init_data) override;
index b47f11443fd2be95333fc601c73fb58cb7261883..f0305e1b2ec59676e7a83c7fa82fcfd6a990e846 100644 (file)
@@ -87,6 +87,7 @@ IPC_STRUCT_TRAITS_MEMBER(side_data_size)
 IPC_STRUCT_TRAITS_MEMBER(side_data)
 IPC_STRUCT_TRAITS_MEMBER(tz_handle)
 IPC_STRUCT_TRAITS_MEMBER(is_key_frame)
+IPC_STRUCT_TRAITS_MEMBER(buffer_id)
 #endif
 IPC_STRUCT_TRAITS_END()
 
@@ -400,6 +401,9 @@ IPC_MESSAGE_CONTROL3(MediaPlayerEflMsg_OnNewEMSSTbmBuffer,
 #endif  // defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+IPC_MESSAGE_ROUTED2(MediaPlayerEflMsg_BufferConsumed,
+                    int /* player_id */,
+                    int32_t /* buffer_id */)
 IPC_SYNC_MESSAGE_ROUTED1_1(MediaPlayerHostMsg_GetDroppedFrameCount,
                            int /* player_id */,
                            unsigned /* droppedFrameCount */)
index fba339076bf5ea9d3b3ddea0ae83b9b0fed7af7f..ac021941cc037cc72c54ff568abfa4d6e93b2abd 100644 (file)
@@ -117,6 +117,7 @@ bool RendererMediaPlayerManager::OnMessageReceived(
                       OnNewTbmBufferAvailable)
 #endif  // defined(TIZEN_TBM_SUPPORT)
 #if defined(OS_TIZEN_TV_PRODUCT)
+  IPC_MESSAGE_HANDLER(MediaPlayerEflMsg_BufferConsumed, OnBufferConsumed)
   IPC_MESSAGE_HANDLER(MediaPlayerEflMsg_IsPlayerStarted, OnPlayerStarted)
   IPC_MESSAGE_HANDLER(MediaPlayerEflMsg_PlaybackComplete, OnPlaybackComplete)
   IPC_MESSAGE_HANDLER(MediaPlayerEflMsg_HardwareResources,
@@ -669,6 +670,16 @@ void RendererMediaPlayerManager::UnregisterMediaPlayer(int player_id) {
 }
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+void RendererMediaPlayerManager::OnBufferConsumed(int player_id,
+                                                  int32_t buffer_id) {
+  auto player = GetMediaPlayer(player_id);
+  if (player) {
+    player->OnBufferConsumed(buffer_id);
+  } else {
+    LOG(INFO) << "Player[" << player_id << "] is already destroyed";
+  }
+}
+
 void RendererMediaPlayerManager::OnPlayerStarted(int player_id,
                                                  bool player_started) {
   auto player = GetMediaPlayer(player_id);
index 78ffa94e3cca8f7f4bc6af24be9317cb25f8f1db..488b27e3e8c564d55c66c29e5e5a6023109154e5 100644 (file)
@@ -113,6 +113,7 @@ class MediaPlayerManagerClient {
                                        base::OnceClosure) {}
 #endif
 #if defined(OS_TIZEN_TV_PRODUCT)
+  virtual void OnBufferConsumed(int32_t buffer_id) {}
   virtual void OnAddAudioTrack(
       const blink::WebMediaPlayer::audio_video_track_info_s&) {}
   virtual void OnAddVideoTrack(
@@ -304,6 +305,7 @@ class RendererMediaPlayerManager : public RenderFrameObserver {
       media_player_mgr_{};
   std::unique_ptr<class MediaPlayerManagerEflImpl> rmgr_impl_;
 
+  void OnBufferConsumed(int player_id, int32_t buffer_id);
   void OnGetPlayTrackInfo(int player_id, int id);
   void OnPlayerStarted(int player_id, bool player_started);
   void OnPlaybackComplete(int player_id, bool send_eos);
index b96866c23730e3c1bb0782696a3424399317cde4..d2b19720a807c778c67a01d7784b4524d2d31513 100644 (file)
@@ -72,6 +72,7 @@ struct MEDIA_EXPORT DemuxedBufferMetaData {
   std::vector<uint8_t> side_data;
   int tz_handle = 0;  // Handle to data decrypted in the trusted zone
   bool is_key_frame = false;
+  int32_t buffer_id = -1;
 #endif
 };
 
index a3340a0553739e2165ba2d63911ca9f354cb482e..ee63719487374afc1b3e5f346eebef8d8e90b795 100644 (file)
@@ -101,6 +101,7 @@ class MEDIA_EXPORT MediaPlayerManager {
   virtual void UpdateZOrder() = 0;
 #endif
 #if defined(OS_TIZEN_TV_PRODUCT)
+  virtual void OnBufferConsumed(int player_id, int32_t buffer_id) = 0;
   virtual content::HardwareResourceHelper* HWHelper() = 0;
   virtual void OnSeekableTimeChange(int player_id,
                                     base::TimeDelta min_time,
index ab3f6f3da8e6f08c8c566a381937cb6ca5137dd9..266dbb19480f38f6bac7f67c27fa6146a4652ffd 100644 (file)
@@ -2931,6 +2931,10 @@ PacketProcessResult MediaSourceESPlusPlayerCapi::ProcessPacket(
   }
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+  if (meta_data.buffer_id != -1) {
+    manager()->OnBufferConsumed(GetPlayerId(), meta_data.buffer_id);
+  }
+
   if (paused_by_buffering_logic_ && seek_state_ == MEDIA_SEEK_NONE) {
     UncheckedUpdateReadyState();
   }
index 3b50335fa863d6c08f86e5e723e59530d0941079..a261843eefaf9eeeac8b8a357c5d55196c0a359a 100644 (file)
@@ -471,6 +471,11 @@ bool SuspendingMediaPlayer::RequestVideoDecodedBuffer() {
 #endif  // defined(TIZEN_TBM_SUPPORT)
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+void SuspendingMediaPlayer::OnBufferConsumed(int player_id, int32_t buffer_id) {
+  if (IsPlayerFullyResumed())
+    manager_->OnBufferConsumed(player_id, buffer_id);
+}
+
 void SuspendingMediaPlayer::OnSeekableTimeChange(int player_id,
                                                  base::TimeDelta min_time,
                                                  base::TimeDelta max_time,
index e04ffb6aac380f9263af9ce8254c4e41069a5cfd..29df2e493c2979523b82675a707947a2b16a38bf 100644 (file)
@@ -246,6 +246,7 @@ class SuspendingMediaPlayer : public MediaPlayerInterfaceEfl,
 #endif  // defined(TIZEN_TBM_SUPPORT)
 
 #if defined(OS_TIZEN_TV_PRODUCT)
+  void OnBufferConsumed(int player_id, int32_t buffer_id) override;
   unsigned GetDroppedFrameCount() const override;
   unsigned GetDecodedFrameCount() const override;
   void OnSeekableTimeChange(int player_id,
index 1ee113932dd16c54496c9a41e809d8a183f6893f..e8218f016f5398a2073803da5d58d71ddd51ee8f 100644 (file)
@@ -114,6 +114,7 @@ MMVideoDecoder::MMVideoDecoder(
     Mode mode,
     InitDoneCb init_done_cb,
     FrameCb frame_cb,
+    BufferConsumedCb buffer_consumed_cb,
     RtcCallType calltype,
     absl::optional<TizenAiZoomSettings> ai_zoom_settings)
     : main_thread_(content::RenderThreadImpl::DeprecatedGetMainTaskRunner()),
@@ -125,6 +126,7 @@ MMVideoDecoder::MMVideoDecoder(
       codec_(codec),
       init_done_(std::move(init_done_cb)),
       frame_cb_(std::move(frame_cb)),
+      buffer_consumed_cb_(std::move(buffer_consumed_cb)),
       ai_zoom_settings_(ai_zoom_settings) {
   TRACE_EVENT0("media", "MMVideoDecoder::MMVideoDecoder");
   MP_LOG(INFO) << __func__ << " calltype: " << static_cast<int>(call_type_);
@@ -203,7 +205,8 @@ MMVideoDecoder::CanPushFrameResult MMVideoDecoder::CanPushFrame()
   }
 }
 
-bool MMVideoDecoder::PushFrame(const uint8_t* data,
+bool MMVideoDecoder::PushFrame(uint32_t buffer_id,
+                               base::UnsafeSharedMemoryRegion shared_memory,
                                std::size_t size,
                                const PendingFrame& info) {
   TRACE_EVENT2("webrtc", "MMVideoDecoder::PushFrame", "frame timestamp",
@@ -224,21 +227,12 @@ bool MMVideoDecoder::PushFrame(const uint8_t* data,
   if (!demuxer_)
     return false;
 
-  if (size <= 0) {
-    LOG(ERROR) << "Frame has no data";
-    return false;
-  }
-
-  auto shared_memory = base::UnsafeSharedMemoryRegion::Create(size);
-  media::DemuxedBufferMetaData meta_data;
-
   if (!shared_memory.IsValid()) {
-    LOG(ERROR) << "Failed to create shared memory region.";
+    LOG(ERROR) << "Invalid shared memory region.";
     return false;
   }
 
-  auto mapping = shared_memory.Map();
-  memcpy(mapping.memory(), data, size);
+  media::DemuxedBufferMetaData meta_data;
 
   if (first_frame_rtp_timestamp_ == 0) {
     first_frame_rtp_timestamp_ = info.timestamp;
@@ -254,6 +248,7 @@ bool MMVideoDecoder::PushFrame(const uint8_t* data,
   meta_data.time_duration = base::TimeDelta::FromMilliseconds(16);
   meta_data.end_of_stream = false;
   // meta_data.is_key_frame = info.is_keyframe;
+  meta_data.buffer_id = info.buffer_id;
 
   MP_DLOG(INFO) << __func__ << " rtc_time: " << info.timestamp
                 << " player time: " << timestamp;
@@ -656,6 +651,12 @@ void MMVideoDecoder::OnNewTbmBufferAvailable(
 #endif
 }
 
+void MMVideoDecoder::OnBufferConsumed(int32_t buffer_id) {
+  if (buffer_consumed_cb_) {
+    buffer_consumed_cb_.Run(buffer_id);
+  }
+}
+
 void MMVideoDecoder::SetCallBackFrameSize(const gfx::Size& size) {
   MP_LOG(INFO) << __func__;
   if (!main_thread_->BelongsToCurrentThread()) {
index 0f6cb5b537c716b72fcfc2bd104fafed28ae6a18..2857b85c8931ebeff5b39d840fbc6c90c85689a7 100644 (file)
@@ -64,6 +64,7 @@ class MMVideoDecoder final : public content::MediaPlayerManagerClient {
   using InitDoneCb = base::OnceCallback<void()>;
   using FrameCb = base::RepeatingCallback<void(scoped_refptr<media::VideoFrame>,
                                                uint32_t orig_timestamp)>;
+  using BufferConsumedCb = base::RepeatingCallback<void(int32_t buffer_id)>;
 
   enum class CanDropFrames { kNo, kYes };
   enum class ForceSubResouces { kNo, kYes };
@@ -76,6 +77,7 @@ class MMVideoDecoder final : public content::MediaPlayerManagerClient {
       Mode mode,
       InitDoneCb init_done_cb,
       FrameCb frame_cb,
+      BufferConsumedCb buffer_consumed_cb,
       RtcCallType calltype,
       absl::optional<TizenAiZoomSettings> ai_zoom_settings = absl::nullopt);
   ~MMVideoDecoder() final;
@@ -93,9 +95,11 @@ class MMVideoDecoder final : public content::MediaPlayerManagerClient {
     bool is_keyframe;
     absl::optional<media::VideoFrameMetadata> metadata{};
     base::TimeDelta player_timestamp{};
+    int32_t buffer_id = -1;
   };
 
-  bool PushFrame(const uint8_t* data,
+  bool PushFrame(uint32_t buffer_id,
+                 base::UnsafeSharedMemoryRegion shared_memory,
                  std::size_t size,
                  const PendingFrame& info);
 
@@ -120,6 +124,7 @@ class MMVideoDecoder final : public content::MediaPlayerManagerClient {
   void OnNewTbmBufferAvailable(const gfx::TbmBufferHandle& tbm_buffer_handle,
                                base::TimeDelta time_delta,
                                base::OnceClosure closure) override;
+  void OnBufferConsumed(int32_t buffer_id) override;
   void EnableTbmBufferCallBack(bool enable);
   void SetCallBackFrameSize(const gfx::Size& size);
 
@@ -203,6 +208,7 @@ class MMVideoDecoder final : public content::MediaPlayerManagerClient {
   bool visible_{true};
   InitDoneCb init_done_{};
   FrameCb frame_cb_{};
+  BufferConsumedCb buffer_consumed_cb_{};
   absl::optional<TizenAiZoomSettings> ai_zoom_settings_;
 
   base::WeakPtrFactory<MMVideoDecoder> weak_factory_{this};
index 936bd30dadf749cc0188e4d02ef60dc978861579..509cb4bba68a44ae540f089535529b548e0b654d 100644 (file)
@@ -70,6 +70,10 @@ class MockMediaPlayerManager : public media::MediaPlayerManager {
                base::TimeDelta timestamp,
                base::OnceClosure release_buffer),
               (override));
+  MOCK_METHOD(void,
+              OnBufferConsumed,
+              (int player_id, int32_t buffer_id),
+              (override));
   MOCK_METHOD(void,
               RegisterMediaPacket,
               (int player_id, TbmHandlePtr tbm_handle),