[EMSS] Fixed error reporting when using TTvd 50/314350/6
authorPiotr Bałut <p.balut@samsung.com>
Wed, 5 Jun 2024 10:49:36 +0000 (12:49 +0200)
committerBot Blink <blinkbot@samsung.com>
Wed, 10 Jul 2024 15:43:12 +0000 (15:43 +0000)
This patch enables error reporting when using EMSS with TTvd:

* In normal latency a new code path is added that passes detailed error
  from HTMLMediaElement to EMSS. Old implementation pushed error info
  directly from EMSS Host service to both webmediaplayer and EMSS
  renderer part independantly. Now without EMSS Host, the closest class
  EMSS is connected can learn about media pipeline error is
  webmediaplayer.

* In low latency webmediaplayer is notified about great deal of errors
  (e.g. decoding errors), as it manages decoded frames. In LL scenario,
  EMSS renderer part will be the source of error information (up to
  decoding errors), notifying media element about the error.

Bug: https://jira-eu.sec.samsung.net/browse/VDWASM-1513
Signed-off-by: Piotr Bałut <p.balut@samsung.com>
Change-Id: I3bb06426abc3158ba9ede78d0e5a0a6fa6ef1efe

13 files changed:
third_party/blink/renderer/core/html/media/html_media_element.cc
third_party/blink/renderer/core/html/media/html_media_element.h
third_party/blink/renderer/core/html/media/media_source_attachment.h
third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.cc
third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.h
tizen_src/chromium_impl/content/renderer/media/tizen/elementary_media_stream_source/control_thread/ms_decoding_stream.cc
tizen_src/chromium_impl/third_party/blink/renderer/modules/elementary_media_stream_source/elementary_media_stream_source.cc
tizen_src/chromium_impl/third_party/blink/renderer/modules/elementary_media_stream_source/elementary_media_stream_source.h
tizen_src/chromium_impl/third_party/blink/renderer/modules/elementary_media_stream_source/elementary_media_stream_source_attachment.h
tizen_src/chromium_impl/third_party/blink/renderer/modules/elementary_media_stream_source/same_thread_elementary_media_stream_source_attachment.cc
tizen_src/chromium_impl/third_party/blink/renderer/modules/elementary_media_stream_source/same_thread_elementary_media_stream_source_attachment.h
tizen_src/chromium_impl/third_party/blink/renderer/platform/elementary_media_stream_source/web_elementary_media_stream_source_dispatcher.cc
tizen_src/chromium_impl/third_party/blink/renderer/platform/elementary_media_stream_source/web_elementary_media_stream_source_dispatcher.h

index 61f4393fb4d75169530d477edb9b886d8d583457..4ade86cb6da1d78db7afd6b054983d77c4d5f118 100644 (file)
@@ -4185,6 +4185,12 @@ void HTMLMediaElement::DurationChanged(double duration, bool request_seek) {
     Seek(duration);
 }
 
+#if defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+void HTMLMediaElement::OnEmssPipelineError(MediaError* error) {
+  MediaEngineError(error);
+}
+#endif  // defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+
 void HTMLMediaElement::RemotePlaybackCompatibilityChanged(const WebURL& url,
                                                           bool is_compatible) {
   if (RemotePlaybackClient())
@@ -5456,7 +5462,7 @@ void HTMLMediaElement::OnLoadDeferredUntilSourceOpen() {
 void HTMLMediaElement::FireDeferredLoadAfterSourceOpen() {
   LOG(INFO) << "FireDeferredLoadAfterSourceOpen(" << *this << ")";
 
-  if (!web_media_player_ || !src_object_stream_descriptor_)
+  if (!web_media_player_ || !src_object_stream_descriptor_ || error_)
     return;
 
   auto source =
@@ -6034,8 +6040,15 @@ void HTMLMediaElement::SetError(MediaError* error) {
   if (error) {
     DLOG(ERROR) << __func__ << ": {code=" << error->code()
                 << ", message=" << error->message() << "}";
-    if (media_source_attachment_)
+    if (media_source_attachment_) {
       media_source_attachment_->OnElementError();
+#if defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+      if (media_source_attachment_->IsElementaryMediaStreamSource()) {
+        media_source_attachment_->OnError(
+            media_source_tracer_, *error);
+      }
+#endif
+    }
   }
 }
 
index 8369db05ae32ad8649a80213beb84d055e6be98d..f942023065492087a54b19f1d0e315a0dafa915d 100644 (file)
@@ -285,6 +285,15 @@ class CORE_EXPORT HTMLMediaElement
   void CloseMediaSource();
   void DurationChanged(double duration, bool request_seek);
 
+#if defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+  // elementary media stream source
+
+  // Used when WebMediaPlayer implementation is not be used to report EMSS
+  // errors. Typically this happens when TTvd is used in low latency and
+  // decoding is managed directly by EMSS.
+  void OnEmssPipelineError(MediaError*);
+#endif
+
   // controls
   bool ShouldShowControls(
       const RecordMetricsBehavior = RecordMetricsBehavior::kDoNotRecord) const;
index 8c2764f1aad739e66fe51dc3011521bbfaa980d9..af3fe52853f68cea80803852cc861cbba67dd90e 100644 (file)
 #include "third_party/blink/renderer/platform/wtf/forward.h"
 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
 
+#if defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+#include "core/html/media/media_error.h"
+#endif  // defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
+
 namespace blink {
 
 #if defined(SAMSUNG_ELEMENTARY_MEDIA_STREAM_SOURCE)
@@ -149,6 +153,7 @@ class CORE_EXPORT MediaSourceAttachment
   virtual bool IsGameMode(MediaSourceTracer* tracer) const = 0;
   virtual bool IsLowLatencyMode(MediaSourceTracer* tracer) const = 0;
   virtual bool IsVideoTextureMode(MediaSourceTracer* tracer) const = 0;
+  virtual void OnError(MediaSourceTracer*, const blink::MediaError&) = 0;
   virtual void OnLoadDeferredUntilSourceOpen(
       MediaSourceTracer* tracer,
       base::OnceClosure on_source_opened) = 0;
index d2bd154df99d59e96c0de71fbd911578893f49e0..9e8b21f42046175e4fbd18c84bfab03b9cc31aaf 100644 (file)
@@ -87,6 +87,9 @@ bool MediaSourceAttachmentSupplement::IsVideoTextureMode(
   return false;
 }
 
+void MediaSourceAttachmentSupplement::OnError(MediaSourceTracer*,
+                                              const blink::MediaError&) {}
+
 void MediaSourceAttachmentSupplement::OnLoadDeferredUntilSourceOpen(
     MediaSourceTracer*,
     base::OnceClosure) {}
index c42c31393200f57cf3e1e9b0f795e5bf5dedf079..823dde6f74afea863352d231fa97f64eb83b8dd3 100644 (file)
@@ -149,6 +149,7 @@ class MediaSourceAttachmentSupplement : public MediaSourceAttachment {
   bool IsGameMode(MediaSourceTracer* tracer) const override;
   bool IsLowLatencyMode(MediaSourceTracer* tracer) const override;
   bool IsVideoTextureMode(MediaSourceTracer* tracer) const override;
+  void OnError(MediaSourceTracer*, const blink::MediaError&) override;
   void OnLoadDeferredUntilSourceOpen(
       MediaSourceTracer* tracer,
       base::OnceClosure on_source_opened) override;
index 657f3f4098630bdd9133b464899046522646b6de..19ab7d59cecdd380d316ed984e8baadc6a642a8d 100644 (file)
@@ -52,9 +52,18 @@ void MsDecodingStream::OnSessionIdChange(uint32_t session_id) {}
 
 void MsDecodingStream::OnClosedCaptions(std::vector<uint8_t> closed_captions) {}
 
-void MsDecodingStream::OnPlayerError(BackendError,
+void MsDecodingStream::OnPlayerError(BackendError error,
                                      std::string message,
-                                     base::OnceClosure on_error_reported) {}
+                                     base::OnceClosure on_error_reported) {
+  EMSS_DEBUG() << error << ", message = " << message;
+
+  if (auto client = client_.lock())
+    client->OnPlayerError(error, message);
+
+  // We're on control thread and we can safely assume that the error was already
+  // reported to JS after the client call above.
+  std::move(on_error_reported).Run();
+}
 
 void MsDecodingStream::OnResumeComplete() {}
 
index 4e90de0d53835aa7d0f68354056301038530e339..97cbaaf3cf61a696dbb0061a512a190459ff0c37 100644 (file)
@@ -15,6 +15,7 @@
 #include "media/base/audio_codecs.h"
 #include "media/base/audio_decoder_config.h"
 #include "media/base/media_track.h"
+#include "media/base/pipeline_status.h"
 #include "media/base/video_decoder_config.h"
 #include "platform/mediastream/media_stream_component.h"
 #include "services/elementary_media_stream_source/public/cpp/logger.h"
@@ -468,6 +469,9 @@ ScriptPromise ElementaryMediaStreamSource::close(ScriptState* script_state) {
       base::BindOnce(
           [](base::WeakPtr<ElementaryMediaStreamSource> ptr,
              OperationCallbackResult result) {
+            EMSS_DEBUG_NO_INSTANCE()
+                << "close() promise resolver, have ptr = " << !!ptr;
+
             if (!ptr)
               return;
 
@@ -481,6 +485,12 @@ ScriptPromise ElementaryMediaStreamSource::close(ScriptState* script_state) {
               return;
             }
 
+            if (!ptr->close_resolver_) {
+              EMSS_DEBUG_NO_INSTANCE()
+                  << "close() promise was already resolved.";
+              return;
+            }
+
             ptr->ResolvePromise(
                 ptr->close_resolver_, result,
                 // This will fail only if Play/Pause/Seek triggered track state
@@ -582,6 +592,9 @@ ScriptPromise ElementaryMediaStreamSource::open(ScriptState* script_state) {
       base::BindOnce(
           [](base::WeakPtr<ElementaryMediaStreamSource> ptr,
              OperationCallbackResult result) {
+            EMSS_DEBUG_NO_INSTANCE()
+                << "open() promise resolver, have ptr = " << !!ptr;
+
             if (!ptr)
               return;
 
@@ -595,6 +608,12 @@ ScriptPromise ElementaryMediaStreamSource::open(ScriptState* script_state) {
               return;
             }
 
+            if (!ptr->open_resolver_) {
+              EMSS_DEBUG_NO_INSTANCE()
+                  << "open() promise was already resolved.";
+              return;
+            }
+
             ptr->ResolvePromise(
                 ptr->open_resolver_, result,
                 // This will fail only if Play/Pause/Seek triggered track state
@@ -832,6 +851,36 @@ void ElementaryMediaStreamSource::OnLoadDeferredUntilSourceOpen(
   load_deferred_until_source_open_cb_ = std::move(on_source_opened);
 }
 
+void ElementaryMediaStreamSource::OnPlayerError(
+    const blink::MediaError& media_error) {
+  using DemuxerMode = content::elementary_media_stream_source::DemuxerMode;
+
+  EMSS_DEBUG() << media_error.code() << ": " << media_error.message();
+
+  if (!web_elementary_media_stream_source_dispatcher_) {
+    EMSS_DEBUG();
+    return;
+  }
+
+  const bool is_ttvd_used =
+      web_elementary_media_stream_source_dispatcher_->pipeline_mode()
+          .demuxer_mode == DemuxerMode::kMediaStream;
+  if (!is_ttvd_used || IsLowLatencyMode()) {
+    // Non-TTvd transmits error directly from EMSS renderer. TTvd low latency
+    // generates error in MsDecodingStream and then passes this info to media
+    // element.
+    EMSS_DEBUG() << "A more precise error info is expected to be delivered by "
+                    "other means and this one should be discarded.";
+    return;
+  }
+
+  const auto player_error =
+      media_error.code() == blink::MediaError::kMediaErrSrcNotSupported
+          ? BackendError::kConfigError
+          : BackendError::kPipelineError;
+  OnPlayerError(player_error, media_error.message().Ascii());
+}
+
 void ElementaryMediaStreamSource::OnPause() {
   EMSS_DEBUG();
 
@@ -1084,6 +1133,12 @@ void ElementaryMediaStreamSource::OnPlayerError(BackendError error,
 
   DispatchEvent(*ElementaryMediaErrorEvent::Create(error, message));
 
+  if (media_stream_descriptor_ && IsLowLatencyMode()) {
+    // When TTvd is used in low latency modes, web media player will NOT make
+    // media element send an error, therefore we trigger it from EMSS:
+    PropagateErrorViaMediaSourceAttachment(error, message);
+  }
+
   Stop();
 }
 
@@ -1304,6 +1359,7 @@ void ElementaryMediaStreamSource::ResolvePromise(
     OperationCallbackResult result,
     const WTF::String& on_error) {
   EMSS_DEBUG_NO_INSTANCE();
+  EMSS_LOG_ASSERT(resolver);
 
   switch (result) {
     case OperationCallbackResult::kSuccess:
@@ -1532,6 +1588,22 @@ void ElementaryMediaStreamSource::OnReadyStateChanged() {
   }
 }
 
+void ElementaryMediaStreamSource::PropagateErrorViaMediaSourceAttachment(
+    BackendError error,
+    const std::string& message) {
+  EMSS_DEBUG() << error << ", " << message;
+
+  if (!emss_attachment_ || !emss_tracer_)
+    return;
+
+  const auto error_code = error == BackendError::kConfigError
+                              ? MediaError::kMediaErrSrcNotSupported
+                              : MediaError::kMediaErrDecode;
+  emss_attachment_->OnEmssPipelineError(
+      emss_tracer_,
+      MakeGarbageCollected<MediaError>(error_code, String{message}));
+}
+
 void ElementaryMediaStreamSource::SetReadyState(ReadyState state) {
   EMSS_LOG(INFO) << "Requested ready state change from " << ready_state_
                  << " to " << state;
index 8602e2da25e2a33f2df4c93a78f252c31107a944..5c5b3946ae12d940c496ebb73333935154e917a8 100644 (file)
@@ -144,6 +144,7 @@ class ElementaryMediaStreamSource final
   bool IsLowLatencyMode() const;
   bool IsVideoTextureMode() const;
   void OnLoadDeferredUntilSourceOpen(base::OnceClosure on_source_opened);
+  void OnPlayerError(const blink::MediaError& media_error);
   void OnPause();
   void OnPlay();
   void OnResume();
@@ -200,6 +201,8 @@ class ElementaryMediaStreamSource final
   bool IsCalledByLifecycleObserver() const;
   bool IsExecutionContextDestroyed() const;
   void OnReadyStateChanged();
+  void PropagateErrorViaMediaSourceAttachment(BackendError,
+                                              const std::string& message);
   void SetReadyState(ReadyState new_state);
 
   static void ResolvePromise(ScriptPromiseResolver* resolver,
index a19689aa1515612894dae271db2d0f4daa436702..19bd6d1f9d1cc4b2d9070fb74b6ab670cec43abe 100644 (file)
@@ -5,6 +5,7 @@
 #ifndef TIZEN_SRC_CHROMIUM_IMPL_THIRD_PARTY_BLINK_RENDERER_MODULES_ELEMENTARY_MEDIA_STREAM_SOURCE_ELEMENTARY_MEDIA_STREAM_SOURCE_ATTACHMENT_H_
 #define TIZEN_SRC_CHROMIUM_IMPL_THIRD_PARTY_BLINK_RENDERER_MODULES_ELEMENTARY_MEDIA_STREAM_SOURCE_ELEMENTARY_MEDIA_STREAM_SOURCE_ATTACHMENT_H_
 
+#include "core/html/media/media_error.h"
 #include "third_party/blink/renderer/core/html/media/media_source_attachment.h"
 
 namespace blink {
@@ -23,6 +24,8 @@ class ElementaryMediaStreamSourceAttachment : public MediaSourceAttachment {
   virtual void OnDurationChanged(MediaSourceTracer* tracer,
                                  double duration,
                                  ShouldSeekToNewDuration should_seek) = 0;
+  virtual void OnEmssPipelineError(MediaSourceTracer* tracer,
+                                   MediaError* error) = 0;
 
   // URLRegistrable interface:
 
index ab6e5b296013d2035077ffc91723e0c99570dc2f..87930e69118ba0c9edd089acb1ba208730fe1991 100644 (file)
@@ -165,6 +165,14 @@ void SameThreadElementaryMediaStreamSourceAttachment::OnPause(
   GetSource(tracer)->OnPause();
 }
 
+void SameThreadElementaryMediaStreamSourceAttachment::OnError(
+    MediaSourceTracer* tracer,
+    const blink::MediaError& media_error) {
+  EMSS_DEBUG();
+
+  GetSource(tracer)->OnPlayerError(media_error);
+}
+
 void SameThreadElementaryMediaStreamSourceAttachment::
     OnLoadDeferredUntilSourceOpen(MediaSourceTracer* tracer,
                                   base::OnceClosure on_source_opened) {
@@ -221,4 +229,12 @@ void SameThreadElementaryMediaStreamSourceAttachment::OnDurationChanged(
   GetMediaElement(tracer)->DurationChanged(duration, seek_to_duration);
 }
 
+void SameThreadElementaryMediaStreamSourceAttachment::OnEmssPipelineError(
+    MediaSourceTracer* tracer,
+    MediaError* error) {
+  EMSS_DEBUG();
+
+  GetMediaElement(tracer)->OnEmssPipelineError(error);
+}
+
 }  // namespace blink
index 6939397b93c227e09cc3cce888cb389d0bc08912..d635cf3989dfcb4c58fca04093444d735927af70 100644 (file)
@@ -58,6 +58,8 @@ class SameThreadElementaryMediaStreamSourceAttachment final
 
   bool IsVideoTextureMode(MediaSourceTracer* tracer) const override;
 
+  void OnError(MediaSourceTracer* tracer, const blink::MediaError&) override;
+
   void OnLoadDeferredUntilSourceOpen(
       MediaSourceTracer* tracer,
       base::OnceClosure on_source_opened) override;
@@ -82,6 +84,8 @@ class SameThreadElementaryMediaStreamSourceAttachment final
   void OnDurationChanged(MediaSourceTracer* tracer,
                          double duration,
                          ShouldSeekToNewDuration) override;
+  void OnEmssPipelineError(MediaSourceTracer* tracer,
+                           MediaError* error) override;
 
  private:
   // This class is created exclusively by URLElementaryMediaStreamSource, which
index 0c0e3032c3a8f3bd953cf5e6321af60add0b95ed..4a99b4a8f440b07fbf6a5c15e0562558cd73a40d 100644 (file)
@@ -22,7 +22,7 @@ WebElementaryMediaStreamSourceDispatcher::Create(
 
   auto blink_source_dispatcher = std::make_shared<
       blink::WebElementaryMediaStreamSourceDispatcher>(
-      control_thread_task_runner, std::move(source_dispatcher),
+      pipeline_mode, control_thread_task_runner, std::move(source_dispatcher),
       std::static_pointer_cast<blink::WebElementaryMediaStreamSourceControl>(
           control_thread_source_impl));
 
@@ -34,10 +34,12 @@ WebElementaryMediaStreamSourceDispatcher::Create(
 
 WebElementaryMediaStreamSourceDispatcher::
     WebElementaryMediaStreamSourceDispatcher(
+        PipelineMode pipeline_mode,
         scoped_refptr<base::SingleThreadTaskRunner> control_thread_task_runner,
         std::shared_ptr<SourceDispatcher> source_dispatcher,
         std::shared_ptr<WebElementaryMediaStreamSourceControl> client)
     : Dispatcher(std::move(control_thread_task_runner)),
+      pipeline_mode_(pipeline_mode),
       source_dispatcher_(std::move(source_dispatcher)) {
   EMSS_DEBUG() << "Constructing object";
 
index 16bee38972c3999e380a7b1613f94ed27a488cd6..06b76db131d9fdc2f6becba752ea05090cea8325 100644 (file)
@@ -64,11 +64,14 @@ class WebElementaryMediaStreamSourceDispatcher
       scoped_refptr<base::SingleThreadTaskRunner> worker_thread_task_runner);
 
   WebElementaryMediaStreamSourceDispatcher(
+      PipelineMode pipeline_mode,
       scoped_refptr<base::SingleThreadTaskRunner> control_thread_task_runner,
       std::shared_ptr<SourceDispatcher> source_dispatcher,
       std::shared_ptr<WebElementaryMediaStreamSourceControl> client);
   ~WebElementaryMediaStreamSourceDispatcher() override;
 
+  const PipelineMode& pipeline_mode() const { return pipeline_mode_; }
+
   std::shared_ptr<WebElementaryMediaTrackDispatcher> CreateAudioTrackDispatcher(
       media::AudioDecoderConfig&& config,
       DecodingMode decoding_mode) {
@@ -107,6 +110,8 @@ class WebElementaryMediaStreamSourceDispatcher
   }
 
  private:
+  const PipelineMode pipeline_mode_;
+
   std::tuple<std::shared_ptr<WebElementaryMediaStreamSourceControl>> clients_;
 
   std::shared_ptr<SourceDispatcher> source_dispatcher_;