[EMSS] Fix software decoded Video Texture when playing in low latency modes

[PROBLEM]
Video frames emitted by software decoders used by MsDecodingStream lack
hardware representation and cannot be rendered in Video Texture mode.

[SOLUTION]
GpuMemoryBufferVideoFramePool is used to create HW representation of
software decoded frames when it is neccessary.

Bug: https://jira-eu.sec.samsung.net/browse/VDWASM-1552
Change-Id: Ib0d6fbd17e9e8bfefda18a03ce96431386c2c699
Signed-off-by: Piotr Bałut <p.balut@samsung.com>

diff --git a/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.cc b/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.cc
index c0a268093f64c7e31739fb4f45e59d1ca95e9df2..346e5ebfacf95bc1906f40c96491b2967ce47180 100644 (file)
--- a/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.cc
+++ b/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.cc
@@ -8,8 +8,11 @@
 
 #include "base/check.h"
 #include "base/functional/bind.h"
+#include "base/functional/callback_forward.h"
 #include "base/functional/callback_helpers.h"
+#include "base/task/bind_post_task.h"
 #include "base/task/sequenced_task_runner.h"
+#include "base/task/thread_pool.h"
 #include "content/renderer/media/tizen/elementary_media_stream_source/common/common.h"
 #include "content/renderer/media/tizen/elementary_media_stream_source/common/logger.h"
 #include "content/renderer/media/tizen/elementary_media_stream_source/control_thread/demuxer_dispatcher_client.h"
@@ -25,6 +28,7 @@
 #include "media/filters/decoder_selector.h"
 #include "media/filters/decrypting_demuxer_stream.h"
 #include "media/filters/tizen/decoder_selection_policy.h"
+#include "media/video/gpu_memory_buffer_video_frame_pool.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "third_party/blink/public/platform/platform.h"
 #include "third_party/blink/renderer/platform/elementary_media_stream_source/web_elementary_media_track_ms_audio_sink.h"
@@ -267,9 +271,14 @@ class MsDecodingStream::DecodingStream {
  public:
   using Traits = DecoderTraits<StreamType>;
   using CreateDecodersCb = typename Traits::CreateDecodersCb;
+  using DecodedBuffer = typename Traits::DecodedBuffer;
   using Sink = typename Traits::Sink;
   using StreamConfig = typename Traits::StreamConfig;
 
+  using ProcessOutputCb = base::RepeatingCallback<void(
+      scoped_refptr<DecodedBuffer>,
+      base::OnceCallback<void(scoped_refptr<DecodedBuffer>)>)>;
+
   DecodingStream(MsDecodingStream* parent,
                  media::MediaLog* media_log,
                  std::shared_ptr<Sink> sink);
@@ -284,7 +293,6 @@ class MsDecodingStream::DecodingStream {
   TrackType Type() const { return StreamType; }
 
  private:
-  using DecodedBuffer = typename Traits::DecodedBuffer;
   using MediaDecoder = typename Traits::MediaDecoder;
 
   bool is_initializing() const { return !!pending_decoder_; }
@@ -308,6 +316,7 @@ class MsDecodingStream::DecodingStream {
   void DestroyCurrentDecoder();
 
   void OnBufferDecoded(scoped_refptr<DecodedBuffer>);
+  void OnBufferDecodedAndProcessed(scoped_refptr<DecodedBuffer>);
   void OnDecoderWaiting(media::WaitingReason);
 
   StreamConfig config_;
@@ -326,6 +335,7 @@ class MsDecodingStream::DecodingStream {
 
   bool is_flushing_until_keyframe_{false};
 
+  ProcessOutputCb process_output_cb_;
   MsDecodingStream* parent_;
   base::WeakPtrFactory<DecodingStream<StreamType>> weak_ptr_factory_{this};
 };  // class DecodingStream
@@ -532,6 +542,9 @@ void MsDecodingStream::DecodingStream<StreamType>::OnDecoderInitialized(
   }
 
   decoder_ = std::move(pending_decoder_);
+  if constexpr (StreamType == TrackType::kVideo) {
+    process_output_cb_ = parent_->GetBufferProcessorForDecoder(*decoder_);
+  }
   TryProcessPendingDecodes();
 }
 
@@ -675,6 +688,23 @@ void MsDecodingStream::DecodingStream<StreamType>::OnBufferDecoded(
     scoped_refptr<DecodedBuffer> buffer) {
   EMSS_VERBOSE_TYPED() << AsHumanReadableString(buffer);
 
+  if (process_output_cb_) {
+    process_output_cb_.Run(
+        std::move(buffer),
+        base::BindPostTaskToCurrentDefault(base::BindOnce(
+            &DecodingStream<StreamType>::OnBufferDecodedAndProcessed,
+            weak_ptr_factory_.GetWeakPtr())));
+    return;
+  }
+
+  OnBufferDecodedAndProcessed(std::move(buffer));
+}
+
+template <TrackType StreamType>
+void MsDecodingStream::DecodingStream<StreamType>::OnBufferDecodedAndProcessed(
+    scoped_refptr<DecodedBuffer> buffer) {
+  EMSS_VERBOSE_TYPED();
+
   sink_->AppendPacket(std::move(buffer));
 }
 
@@ -914,4 +944,39 @@ void MsDecodingStream::OnNewCreateVideoDecodersPrerequisiteAvailable() {
                                  std::move(control_task_runner)));
 }
 
+MsDecodingStream::ProcessVideoOutputCb
+MsDecodingStream::GetBufferProcessorForDecoder(
+    const media::VideoDecoder& video_decoder) {
+  EMSS_VERBOSE();
+
+  if (video_decoder.IsPlatformDecoder()) {
+    return {};
+  }
+
+  if (!gmb_pool_) {
+    EMSS_VERBOSE();
+    EMSS_LOG_ASSERT(gpu_factories_) << "GpuFactories should've been set by now";
+
+    gmb_pool_ = std::make_unique<media::GpuMemoryBufferVideoFramePool>(
+        base::SequencedTaskRunner::GetCurrentDefault(),
+        base::ThreadPool::CreateSequencedTaskRunner(
+            {base::TaskPriority::USER_BLOCKING}),
+        gpu_factories_);
+  }
+
+  return base::BindPostTaskToCurrentDefault(
+      base::BindRepeating(&MsDecodingStream::ProcessSoftwareDecodedVideoFrame,
+                          weak_ptr_factory_.GetWeakPtr()));
+}
+
+void MsDecodingStream::ProcessSoftwareDecodedVideoFrame(
+    scoped_refptr<media::VideoFrame> frame,
+    VideoOutputProcessedCb on_frame_processed) {
+  EMSS_VERBOSE();
+  EMSS_LOG_ASSERT(gmb_pool_);
+
+  gmb_pool_->MaybeCreateHardwareFrame(std::move(frame),
+                                      std::move(on_frame_processed));
+}
+
 }  // namespace content::elementary_media_stream_source::worker_thread

diff --git a/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.h b/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.h
index 07ff182c9007198631275caf35dcf4979e542e7e..5733017851184a6825a40fc2aa22b8b03b937d71 100644 (file)
--- a/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.h
+++ b/tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/worker_thread/ms_decoding_stream.h
@@ -14,6 +14,7 @@
 #include "content/renderer/media/tizen/elementary_media_stream_source/worker_thread/platform_demuxer_adapter_client.h"
 #include "media/base/media_log.h"
 #include "media/base/video_decoder.h"
+#include "media/base/video_frame.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 
 namespace blink {
@@ -23,6 +24,7 @@ class WebElementaryMediaTrackMsVideoSink;
 
 namespace media {
 class AudioDecoder;
+class GpuMemoryBufferVideoFramePool;
 class GpuVideoAcceleratorFactories;
 }  // namespace media
 
@@ -95,15 +97,27 @@ class MsDecodingStream : public Demuxer,
   using AudioStream = DecodingStream<TrackType::kAudio>;
   using VideoStream = DecodingStream<TrackType::kVideo>;
 
+  using VideoOutputProcessedCb =
+      base::OnceCallback<void(scoped_refptr<media::VideoFrame>)>;
+  using ProcessVideoOutputCb =
+      base::RepeatingCallback<void(scoped_refptr<media::VideoFrame>,
+                                   VideoOutputProcessedCb)>;
+
   void OnNewCreateAudioDecodersPrerequisiteAvailable();
   void OnNewCreateVideoDecodersPrerequisiteAvailable();
 
+  ProcessVideoOutputCb GetBufferProcessorForDecoder(const media::VideoDecoder&);
+  void ProcessSoftwareDecodedVideoFrame(
+      scoped_refptr<media::VideoFrame> frame,
+      VideoOutputProcessedCb on_frame_processed);
+
   std::unique_ptr<AudioStream> audio_stream_;
   std::unique_ptr<VideoStream> video_stream_;
 
   gfx::ColorSpace target_color_space_;
   base::WeakPtr<media::DecoderFactory> decoder_factory_;
   std::weak_ptr<any_thread::DemuxerDispatcher> dispatcher_;
+  std::unique_ptr<media::GpuMemoryBufferVideoFramePool> gmb_pool_;
   media::GpuVideoAcceleratorFactories* gpu_factories_{nullptr};
   std::unique_ptr<media::MediaLog> media_log_;