[TTVD] Report real memory usage statistics by decoder

Our implementation of hardware video decoder for Tizen TV
platform can operate without allocating memory for video
frames. However, this was not respected by video renderer
that calculated memory usage using buffered video frames
coded size.

This CL introduces new mechanism to report actual memory
usage by frame pool inside hardware decoder implementation.
Video renderer registers for callback and receives update
each time new backing is created/destroyed. With that, we
can now properly report memory usage in overlay mode.

This also helps reducing frequency of garbage collector
being called due to external memory exceeding the limit.

Bug: https://jira-eu.sec.samsung.net/browse/VDGAME-680
Change-Id: I0a292fafc8a2754201e2cce92807b8cdee48f24d
Signed-off-by: Jakub Gajownik <j.gajownik2@samsung.com>

diff --git a/media/filters/tizen/ttvd_decoded_frame.cc b/media/filters/tizen/ttvd_decoded_frame.cc
index ac8ad21f5fb9b3a30989e431e5ecf80e6ab707f2..fbee17007f4ce0abd54c7bafa60328a40a5568bd 100644 (file)
--- a/media/filters/tizen/ttvd_decoded_frame.cc
+++ b/media/filters/tizen/ttvd_decoded_frame.cc
@@ -32,10 +32,12 @@ void TTvdDecodedFrame::CreateAsync(
                                  OverlayRenderCb,
                                  base::OnceClosure)> render_cb,
     base::RepeatingCallback<void(scoped_refptr<TTvdDecodedFrame>)> release_cb,
+    MemoryUsageChangeCB memory_usage_cb,
     const gpu::GpuDriverBugWorkarounds* workarounds) {
-  scoped_refptr<TTvdDecodedFrame> result(new TTvdDecodedFrame(
-      natural_size, std::move(command_buffer_helper),
-      std::move(gpu_task_runner), std::move(release_cb), workarounds));
+  scoped_refptr<TTvdDecodedFrame> result(
+      new TTvdDecodedFrame(natural_size, std::move(command_buffer_helper),
+                           std::move(gpu_task_runner), std::move(release_cb),
+                           memory_usage_cb, workarounds));
   TTvdDecodedFrame* raw_result = result.get();
   raw_result->InitializeGpuPart(
       allocate_memory, plane_collection,
@@ -48,16 +50,23 @@ TTvdDecodedFrame::TTvdDecodedFrame(
     scoped_refptr<CommandBufferHelper> command_buffer_helper,
     scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner,
     base::RepeatingCallback<void(scoped_refptr<TTvdDecodedFrame>)> release_cb,
+    MemoryUsageChangeCB memory_usage_cb,
     const gpu::GpuDriverBugWorkarounds* workarounds)
     : picture_size_(natural_size),
       command_buffer_helper_(std::move(command_buffer_helper)),
       gpu_task_runner_(std::move(gpu_task_runner)),
       release_cb_(std::move(release_cb)),
+      memory_usage_cb_(std::move(memory_usage_cb)),
       workarounds_(workarounds) {
   weak_this_ = weak_factory_.GetWeakPtr();
 }
 
 TTvdDecodedFrame::~TTvdDecodedFrame() {
+  if (pixmap_) {
+    const auto allocation_size =
+        VideoFrame::AllocationSize(PIXEL_FORMAT_NV12, picture_size_);
+    memory_usage_cb_.Run(-static_cast<int64_t>(allocation_size));
+  }
   weak_factory_.InvalidateWeakPtrs();
   gpu_task_runner_->DeleteSoon(FROM_HERE, std::move(frame_on_gpu_));
 }
@@ -129,6 +138,11 @@ void TTvdDecodedFrame::InitializeGpuPart(
     base::RepeatingCallback<void(base::WeakPtr<TTvdDecodedFrame>,
                                  OverlayRenderCb,
                                  base::OnceClosure)> render_cb) {
+  if (allocate_memory) {
+    const auto allocation_size =
+        VideoFrame::AllocationSize(PIXEL_FORMAT_NV12, picture_size_);
+    memory_usage_cb_.Run(static_cast<int64_t>(allocation_size));
+  }
   gpu_task_runner_->PostTask(
       FROM_HERE,
       base::BindOnce(
@@ -201,6 +215,11 @@ void TTvdDecodedFrame::RecreateBackingForTexture(bool clear_black) {
     return;
   }
 
+  if (!pixmap_) {
+    const auto allocation_size =
+        VideoFrame::AllocationSize(PIXEL_FORMAT_NV12, picture_size_);
+    memory_usage_cb_.Run(static_cast<int64_t>(allocation_size));
+  }
   pixmap_ = std::move(allocation_result);
 }
 

diff --git a/media/filters/tizen/ttvd_decoded_frame.h b/media/filters/tizen/ttvd_decoded_frame.h
index c9511f183513c43c110aecaacbaedcd530066221..5e2f83861de57487b73d089ccc13b15b9eea9a7e 100644 (file)
--- a/media/filters/tizen/ttvd_decoded_frame.h
+++ b/media/filters/tizen/ttvd_decoded_frame.h
@@ -36,6 +36,7 @@ class TTvdDecodedFrame : public base::RefCountedThreadSafe<TTvdDecodedFrame> {
   using OverlayRenderCb =
       base::RepeatingCallback<void(gfx::OverlayRenderData,
                                    base::OnceCallback<void(bool)>)>;
+  using MemoryUsageChangeCB = base::RepeatingCallback<void(int64_t)>;
   static void CreateAsync(
       gfx::Size natural_size,
       bool allocate_memory,
@@ -47,6 +48,7 @@ class TTvdDecodedFrame : public base::RefCountedThreadSafe<TTvdDecodedFrame> {
                                    OverlayRenderCb,
                                    base::OnceClosure)> render_cb,
       base::RepeatingCallback<void(scoped_refptr<TTvdDecodedFrame>)> release_cb,
+      MemoryUsageChangeCB memory_usage_cb,
       const gpu::GpuDriverBugWorkarounds* workarounds);
   ~TTvdDecodedFrame();
 
@@ -83,6 +85,7 @@ class TTvdDecodedFrame : public base::RefCountedThreadSafe<TTvdDecodedFrame> {
       scoped_refptr<CommandBufferHelper> command_buffer_helper,
       scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner,
       base::RepeatingCallback<void(scoped_refptr<TTvdDecodedFrame>)> release_cb,
+      MemoryUsageChangeCB memory_usage_cb,
       const gpu::GpuDriverBugWorkarounds* workarounds);
 
   void VideoFrameDestroyed(const gpu::SyncToken& sync_token);
@@ -131,6 +134,8 @@ class TTvdDecodedFrame : public base::RefCountedThreadSafe<TTvdDecodedFrame> {
   std::unique_ptr<TTvdDecodedFrameOnGpu> frame_on_gpu_;
   base::RepeatingCallback<void(scoped_refptr<TTvdDecodedFrame>)> release_cb_;
 
+  MemoryUsageChangeCB memory_usage_cb_;
+
   const gpu::GpuDriverBugWorkarounds* workarounds_;
 
   base::WeakPtr<TTvdDecodedFrame> weak_this_;

diff --git a/media/filters/tizen/ttvd_video_decoder.cc b/media/filters/tizen/ttvd_video_decoder.cc
index 33e03037eab3e57789af89edaba0e297ff8e976b..f38f05e2086841ba0875e68a3c62a87f2c344846 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder.cc
+++ b/media/filters/tizen/ttvd_video_decoder.cc
@@ -99,6 +99,13 @@ void TTvdVideoDecoder::OnVideoFrameDecoded(
   output_cb_.Run(frame);
 }
 
+void TTvdVideoDecoder::OnTotalMemoryUsageReport(int64_t total_video_memory) {
+  last_reported_memory_usage_ = total_video_memory;
+  if (total_memory_usage_cb_) {
+    total_memory_usage_cb_.Run(last_reported_memory_usage_);
+  }
+}
+
 void TTvdVideoDecoder::OnWaiting(WaitingReason reason) {
   waiting_cb_.Run(reason);
 }
@@ -109,6 +116,14 @@ void TTvdVideoDecoder::BindVideoResourceHelper(
   decoder_->BindVideoResourceHelper(std::move(receiver), std::move(client));
 }
 
+void TTvdVideoDecoder::SetTotalMemoryUsageCB(
+    TotalMemoryUsageCB memory_usage_cb) {
+  total_memory_usage_cb_ = memory_usage_cb;
+  if (last_reported_memory_usage_) {
+    total_memory_usage_cb_.Run(last_reported_memory_usage_);
+  }
+}
+
 void TTvdVideoDecoder::ReleaseVideoFrame(
     const base::UnguessableToken& release_token,
     const gpu::SyncToken& release_sync_token) {

diff --git a/media/filters/tizen/ttvd_video_decoder.h b/media/filters/tizen/ttvd_video_decoder.h
index b9258c69da8729ab90129f7a57303a7950dde742..b3c8cf5fbe5c48bed6e82a67c610d0175e6f3da1 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder.h
+++ b/media/filters/tizen/ttvd_video_decoder.h
@@ -53,12 +53,16 @@ class MEDIA_EXPORT TTvdVideoDecoder : public VideoDecoder,
   void OnVideoFrameDecoded(
       const scoped_refptr<VideoFrame>& frame,
       const base::UnguessableToken& release_token) override;
+  void OnTotalMemoryUsageReport(int64_t total_video_memory) override;
   void OnWaiting(WaitingReason reason) override;
 
   void BindVideoResourceHelper(
       mojo::PendingAssociatedReceiver<mojom::VideoResourceHelper> receiver,
       mojo::PendingAssociatedRemote<mojom::VideoResourceHelperClient> client);
 
+  using TotalMemoryUsageCB = base::RepeatingCallback<void(int64_t)>;
+  void SetTotalMemoryUsageCB(TotalMemoryUsageCB total_memory_usage_cb);
+
  private:
   void ReleaseVideoFrame(const base::UnguessableToken& release_token,
                          const gpu::SyncToken& release_sync_token);
@@ -95,6 +99,9 @@ class MEDIA_EXPORT TTvdVideoDecoder : public VideoDecoder,
   base::flat_map<uint64_t, DecodeCB> pending_decodes_;
   base::OnceClosure reset_cb_;
 
+  int64_t last_reported_memory_usage_ = 0;
+  TotalMemoryUsageCB total_memory_usage_cb_;
+
   uint32_t writer_capacity_ = 0;
 
   mojo::AssociatedReceiver<mojom::TTvdVideoDecoderClient> client_receiver_{

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.cc b/media/filters/tizen/ttvd_video_decoder_impl.cc
index 02cfe9be0aec7e4a6898dd3a8833b55c99b44898..5a7f2e8f3df7cce925b54f04dd84965a1e8affdd 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.cc
+++ b/media/filters/tizen/ttvd_video_decoder_impl.cc
@@ -1595,6 +1595,8 @@ scoped_refptr<TTvdDecodedFrame> TTvdVideoDecoderImpl::CreateTTvdDecodedFrame() {
                           weak_factory_.GetWeakPtr()),
       base::BindRepeating(&TTvdVideoDecoderImpl::ReleasePictureImpl,
                           weak_factory_.GetWeakPtr()),
+      base::BindRepeating(&TTvdVideoDecoderImpl::OnMemoryUsageChange,
+                          weak_factory_.GetWeakPtr()),
       &workarounds_);
   event.Wait();
   TIZEN_MEDIA_LOG(VERBOSE) << "Decoding results: " << decoding_results_.size()
@@ -1808,6 +1810,8 @@ void TTvdVideoDecoderImpl::CreateLazyTTvdDecodedFrame() {
                           weak_factory_.GetWeakPtr()),
       base::BindRepeating(&TTvdVideoDecoderImpl::ReleasePictureImpl,
                           weak_factory_.GetWeakPtr()),
+      base::BindRepeating(&TTvdVideoDecoderImpl::OnMemoryUsageChange,
+                          weak_factory_.GetWeakPtr()),
       &workarounds_);
 }
 
@@ -2586,6 +2590,13 @@ void TTvdVideoDecoderImpl::TriggerEos() {
   std::move(eos_cb_).Run(DecoderStatus::Codes::kOk);
 }
 
+void TTvdVideoDecoderImpl::OnMemoryUsageChange(int64_t memory_delta) {
+  TIZEN_MEDIA_LOG(VERBOSE) << "Video memory delta: " << memory_delta;
+  total_memory_usage_ += memory_delta;
+  TIZEN_MEDIA_LOG(VERBOSE) << "Total memory usage: " << total_memory_usage_;
+  client_->OnTotalMemoryUsageReport(total_memory_usage_);
+}
+
 void TTvdVideoDecoderImpl::UpdateConfigUsingBuffer(
     const DecoderBuffer& buffer) {
   UpdateConfig(&config_,

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.h b/media/filters/tizen/ttvd_video_decoder_impl.h
index a57f4fbbb39dc1a53d02833911c31e752bac41cf..c6eeef10dc7e12f53177fe34dc1cd4c2abf178ad 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.h
+++ b/media/filters/tizen/ttvd_video_decoder_impl.h
@@ -320,6 +320,9 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl
   // in |decoding_requests_|.
   void TriggerEos();
 
+  // Updates amount of memory usage by decoded frames currently hold by decoder.
+  void OnMemoryUsageChange(int64_t memory_delta);
+
   void UpdateConfigUsingBuffer(const DecoderBuffer& buffer);
   void UpdateConfigFromExtradata(const std::vector<uint8_t>& buffer);
 
@@ -383,6 +386,10 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl
   WaitingCB waiting_cb_;
   base::OnceClosure reset_cb_;
 
+  // Sum of all memory allocations for decoded video data used by decoder
+  // instance.
+  int64_t total_memory_usage_ = 0;
+
   // Pool for not currently used TTvdDecodedFrame instances. Entries
   // should be compatible with |output_mode_|.
   std::deque<scoped_refptr<TTvdDecodedFrame>> ttvd_decoded_frame_pool_;

diff --git a/media/filters/tizen/ttvd_video_renderer.cc b/media/filters/tizen/ttvd_video_renderer.cc
index 504b04b171337813c38719c70355c2c2a67b5106..8886d979d348f185fd7823a64f9e934892686b0e 100644 (file)
--- a/media/filters/tizen/ttvd_video_renderer.cc
+++ b/media/filters/tizen/ttvd_video_renderer.cc
@@ -565,17 +565,26 @@ void TTvdVideoRenderer::OnFallback(PipelineStatus status) {
 
 void TTvdVideoRenderer::OnDecoderChange(VideoDecoder* decoder) {
   if (!decoder) {
+    last_reported_memory_usage_.reset();
     return;
   }
 
-  if (video_resource_strategy_ !=
-          VideoResourceStrategy::kHardwareOnlyAndPause ||
-      !decoder->IsPlatformDecoder()) {
+  if (!decoder->IsPlatformDecoder()) {
     return;
   }
 
   TTvdVideoDecoder* ttvd_video_decoder =
       reinterpret_cast<TTvdVideoDecoder*>(decoder);
+
+  last_reported_memory_usage_ = 0;
+  ttvd_video_decoder->SetTotalMemoryUsageCB(base::BindRepeating(
+      &TTvdVideoRenderer::OnTotalMemoryUsageReport, base::Unretained(this)));
+
+  if (video_resource_strategy_ !=
+      VideoResourceStrategy::kHardwareOnlyAndPause) {
+    return;
+  }
+
   video_resource_helper_.reset();
   video_resource_helper_receiver_.reset();
   ttvd_video_decoder->BindVideoResourceHelper(
@@ -1174,7 +1183,12 @@ void TTvdVideoRenderer::UpdateStats_Locked(bool force_update) {
                          stats_.video_frames_dropped, "id", player_id_);
   }
 
-  const size_t memory_usage = algorithm_->GetMemoryUsage();
+  // If we got data provided directly by hardware decoder, prefer it over
+  // estimation provided by algorithm. It counts real memory allocations
+  // and properly handles HW protected frames.
+  const size_t memory_usage = last_reported_memory_usage_
+                                  ? *last_reported_memory_usage_
+                                  : algorithm_->GetMemoryUsage();
   stats_.video_memory_usage = memory_usage - stats_.video_memory_usage;
   stats_.video_frame_duration_average = algorithm_->average_frame_duration();
   OnStatisticsUpdate(stats_);
@@ -1421,4 +1435,9 @@ void TTvdVideoRenderer::PaintFirstFrame() {
   paint_first_frame_cb_.Cancel();
 }
 
+void TTvdVideoRenderer::OnTotalMemoryUsageReport(int64_t total_video_memory) {
+  DCHECK(task_runner_->RunsTasksInCurrentSequence());
+  last_reported_memory_usage_ = total_video_memory;
+}
+
 }  // namespace media

diff --git a/media/filters/tizen/ttvd_video_renderer.h b/media/filters/tizen/ttvd_video_renderer.h
index 73c8e20a5a152b1712b93f9486c04adbfaa63c5e..804c294f461a744e17907506958b5f1259608a52 100644 (file)
--- a/media/filters/tizen/ttvd_video_renderer.h
+++ b/media/filters/tizen/ttvd_video_renderer.h
@@ -249,6 +249,9 @@ class MEDIA_EXPORT TTvdVideoRenderer
   // `painted_first_frame_` is false.
   void PaintFirstFrame();
 
+  // Called by decoder to indicate new total memory used for held video frames.
+  void OnTotalMemoryUsageReport(int64_t total_video_memory);
+
   scoped_refptr<base::SequencedTaskRunner> task_runner_;
 
   // Sink which calls into TTvdVideoRenderer via Render() for video frames.  Do
@@ -419,6 +422,10 @@ class MEDIA_EXPORT TTvdVideoRenderer
   // Last FPS, if any, reported to the client.
   absl::optional<int> last_reported_fps_;
 
+  // Last reported memory usage by Tizen TV video decoder.
+  // When set, it's prefered when sending pipeline statistics.
+  absl::optional<int64_t> last_reported_memory_usage_;
+
   // Value saved from last call to SetLatencyHint(). Used to recompute buffering
   // limits as framerate fluctuates.
   absl::optional<base::TimeDelta> latency_hint_;

diff --git a/media/mojo/mojom/ttvd_media_provider.mojom b/media/mojo/mojom/ttvd_media_provider.mojom
index d58ddb2a8a49595fc56c385f53a25d002a0f702c..73476d9852f259681884562c520bdeaf00cc1dfd 100644 (file)
--- a/media/mojo/mojom/ttvd_media_provider.mojom
+++ b/media/mojo/mojom/ttvd_media_provider.mojom
@@ -42,6 +42,10 @@ interface TTvdVideoDecoderClient {
   OnVideoFrameDecoded(VideoFrame frame,
                       mojo_base.mojom.UnguessableToken release_token);
 
+  // Provides information what is total memory used by decoder to hold video
+  // frames.
+  OnTotalMemoryUsageReport(int64 total_video_memory);
+
   OnWaiting(WaitingReason reason);
 };