2 * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 #ifndef THIRD_PARTY_BLINK_RENDERER_CORE_HTML_MEDIA_HTML_MEDIA_ELEMENT_H_
28 #define THIRD_PARTY_BLINK_RENDERER_CORE_HTML_MEDIA_HTML_MEDIA_ELEMENT_H_
32 #include "base/synchronization/lock.h"
33 #include "base/thread_annotations.h"
34 #include "base/time/time.h"
35 #include "base/timer/elapsed_timer.h"
36 #include "media/mojo/mojom/media_player.mojom-blink.h"
37 #include "third_party/abseil-cpp/absl/types/optional.h"
38 #include "third_party/abseil-cpp/absl/types/variant.h"
39 #include "third_party/blink/public/common/media/display_type.h"
40 #include "third_party/blink/public/platform/web_media_player_client.h"
41 #include "third_party/blink/public/platform/webaudiosourceprovider_impl.h"
42 #include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
43 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
44 #include "third_party/blink/renderer/core/core_export.h"
45 #include "third_party/blink/renderer/core/execution_context/execution_context_lifecycle_state_observer.h"
46 #include "third_party/blink/renderer/core/html/html_element.h"
47 #include "third_party/blink/renderer/core/html/media/media_controls.h"
48 #include "third_party/blink/renderer/core/intersection_observer/intersection_observer.h"
49 #include "third_party/blink/renderer/core/speech/speech_synthesis_base.h"
50 #include "third_party/blink/renderer/platform/audio/audio_source_provider.h"
51 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
52 #include "third_party/blink/renderer/platform/heap/disallow_new_wrapper.h"
53 #include "third_party/blink/renderer/platform/heap/prefinalizer.h"
54 #include "third_party/blink/renderer/platform/media/web_audio_source_provider_client.h"
55 #include "third_party/blink/renderer/platform/mojo/heap_mojo_associated_receiver_set.h"
56 #include "third_party/blink/renderer/platform/mojo/heap_mojo_associated_remote.h"
57 #include "third_party/blink/renderer/platform/mojo/heap_mojo_associated_remote_set.h"
58 #include "third_party/blink/renderer/platform/network/mime/mime_type_registry.h"
59 #include "third_party/blink/renderer/platform/scheduler/public/post_cancellable_task.h"
60 #include "third_party/blink/renderer/platform/supplementable.h"
61 #include "third_party/blink/renderer/platform/timer.h"
62 #include "third_party/blink/renderer/platform/weborigin/kurl.h"
63 #include "third_party/blink/renderer/platform/wtf/vector.h"
64 #include "third_party/webrtc_overrides/low_precision_timer.h"
71 enum class MediaContentType;
75 #if BUILDFLAG(IS_TIZEN_TV)
77 ADD_INBAND_TEXT_TRACK = 0,
78 DEL_INBAND_TEXT_TRACK,
82 class AudioSourceProviderClient;
91 class HTMLMediaElementControlsList;
92 class HTMLSourceElement;
93 class HTMLTrackElement;
95 class MediaSourceAttachment;
96 class MediaSourceHandle;
97 class MediaSourceTracer;
98 class MediaStreamDescriptor;
99 class ScriptPromiseResolver;
102 class TextTrackContainer;
106 class VideoTrackList;
107 class WebRemotePlaybackClient;
109 class CORE_EXPORT HTMLMediaElement
110 : public HTMLElement,
111 public Supplementable<HTMLMediaElement>,
112 public ActiveScriptWrappable<HTMLMediaElement>,
113 public ExecutionContextLifecycleStateObserver,
114 public media::mojom::blink::MediaPlayer,
115 private WebMediaPlayerClient {
116 DEFINE_WRAPPERTYPEINFO();
117 USING_PRE_FINALIZER(HTMLMediaElement, Dispose);
120 // Limits the range of media playback rate.
121 static constexpr double kMinPlaybackRate = 0.0625;
122 static constexpr double kMaxPlaybackRate = 16.0;
124 enum class PlayPromiseError {
128 kPaused_EndOfPlayback,
129 kPaused_RemovedFromDocument,
130 kPaused_AutoplayAutoPause,
131 kPaused_BackgroundVideoOptimization,
132 kPaused_SuspendedPlayerIdleTimeout,
133 kPaused_RemotePlayStateChange,
134 kPaused_PauseRequestedByUser,
135 kPaused_PauseRequestedInternally,
138 bool IsMediaElement() const override { return true; }
140 static MIMETypeRegistry::SupportsType GetSupportsType(const ContentType&);
142 enum class RecordMetricsBehavior { kDoNotRecord, kDoRecord };
144 static bool IsHLSURL(const KURL&);
146 // Notify the HTMLMediaElement that the media controls settings have changed
147 // for the given document.
148 static void OnMediaControlsEnabledChange(Document*);
150 // Binds |pending_receiver| and adds it to |media_player_receiver_set_|. Also
151 // called from other Blink classes (e.g. PictureInPictureControllerImpl).
152 void BindMediaPlayerReceiver(
153 mojo::PendingAssociatedReceiver<media::mojom::blink::MediaPlayer>
156 void Trace(Visitor*) const override;
158 WebMediaPlayer* GetWebMediaPlayer() const { return web_media_player_.get(); }
160 // Returns true if the loaded media has a video track.
161 // Note that even an audio element can have video track in cases such as
162 // <audio src="video.webm">, in which case this function will return true.
163 bool HasVideo() const override;
165 // Returns true if loaded media has an audio track.
166 bool HasAudio() const;
168 #if BUILDFLAG(IS_TIZEN_TV)
169 bool IsPlaying() const override { return playing_; }
172 // Whether the media element has encrypted audio or video streams.
173 bool IsEncrypted() const;
175 bool SupportsSave() const;
176 bool SupportsLoop() const;
178 cc::Layer* CcLayer() const;
180 enum DelayedActionType {
181 kLoadMediaResource = 1 << 0,
182 kLoadTextTrackResource = 1 << 1
184 void ScheduleTextTrackResourceLoad();
187 MediaError* error() const;
190 void SetSrc(const AtomicString&);
191 const KURL& currentSrc() const { return current_src_.GetSourceIfVisible(); }
193 // Return the URL to be used for downloading the media.
194 const KURL& downloadURL() const {
195 // If we didn't get a redirected URL from the player, then use the original.
196 if (current_src_after_redirects_.IsNull() ||
197 current_src_after_redirects_.IsEmpty()) {
200 return current_src_after_redirects_;
203 using SrcObjectVariant =
204 absl::variant<MediaStreamDescriptor*, MediaSourceHandle*>;
205 void SetSrcObjectVariant(SrcObjectVariant src_object_variant);
206 SrcObjectVariant GetSrcObjectVariant() const;
214 NetworkState getNetworkState() const;
216 String preload() const;
217 void setPreload(const AtomicString&);
218 WebMediaPlayer::Preload PreloadType() const;
219 String EffectivePreload() const;
220 WebMediaPlayer::Preload EffectivePreloadType() const;
222 WebTimeRanges BufferedInternal() const;
223 TimeRanges* buffered() const;
225 String canPlayType(const String& mime_type) const;
235 ReadyState getReadyState() const;
236 bool seeking() const;
239 double currentTime() const;
240 void setCurrentTime(double);
241 double duration() const;
243 double defaultPlaybackRate() const;
244 void setDefaultPlaybackRate(double);
245 double playbackRate() const;
246 void setPlaybackRate(double, ExceptionState& = ASSERT_NO_EXCEPTION);
247 TimeRanges* played();
248 WebTimeRanges SeekableInternal() const;
249 TimeRanges* seekable() const;
251 bool Autoplay() const;
254 ScriptPromise playForBindings(ScriptState*);
255 absl::optional<DOMExceptionCode> Play();
256 double getStartDate() const;
258 // Called when the video should pause to let audio descriptions finish.
259 void PauseToLetDescriptionFinish();
262 double latencyHint() const;
263 void setLatencyHint(double);
264 bool preservesPitch() const;
265 void setPreservesPitch(bool);
266 void FlingingStarted();
267 void FlingingStopped();
270 uint64_t webkitAudioDecodedByteCount() const;
271 uint64_t webkitVideoDecodedByteCount() const;
273 // media source extensions
274 void CloseMediaSource();
275 void DurationChanged(double duration, bool request_seek);
278 bool ShouldShowControls(
279 const RecordMetricsBehavior = RecordMetricsBehavior::kDoNotRecord) const;
280 bool ShouldShowAllControls() const;
281 DOMTokenList* controlsList() const;
282 HTMLMediaElementControlsList* ControlsListInternal() const;
283 double volume() const;
284 void setVolume(double, ExceptionState& = ASSERT_NO_EXCEPTION);
287 virtual bool SupportsPictureInPicture() const { return false; }
288 void SetUserWantsControlsVisible(bool visible);
289 bool UserWantsControlsVisible() const;
291 void TogglePlayState();
293 AudioTrackList& audioTracks();
294 void AudioTrackChanged(AudioTrack*);
296 VideoTrackList& videoTracks();
297 void SelectedVideoTrackChanged(VideoTrack*);
299 TextTrack* addTextTrack(const AtomicString& kind,
300 const AtomicString& label,
301 const AtomicString& language,
304 TextTrackList* textTracks();
305 CueTimeline& GetCueTimeline();
307 // Implements the "forget the media element's media-resource-specific tracks"
308 // algorithm in the HTML5 spec.
309 void ForgetResourceSpecificTracks();
311 void DidAddTrackElement(HTMLTrackElement*);
312 void DidRemoveTrackElement(HTMLTrackElement*);
314 void HonorUserPreferencesForAutomaticTextTrackSelection();
316 bool TextTracksAreReady() const;
317 void ConfigureTextTrackDisplay();
318 void UpdateTextTrackDisplay();
320 // Get a SpeechSynthesis interface to use for generating speech for audio
322 SpeechSynthesisBase* SpeechSynthesis();
323 double LastSeekTime() const { return last_seek_time_; }
324 void TextTrackReadyStateChanged(TextTrack*);
326 void TextTrackModeChanged(TextTrack*);
327 void DisableAutomaticTextTrackSelection();
329 // EventTarget function.
330 // Both Node (via HTMLElement) and ExecutionContextLifecycleStateObserver
331 // define this method, which causes an ambiguity error at compile time. This
332 // class's constructor ensures that both implementations return document, so
333 // return the result of one of them here.
334 using HTMLElement::GetExecutionContext;
336 bool IsFullscreen() const;
337 virtual bool UsesOverlayFullscreenVideo() const { return false; }
339 bool HasClosedCaptions() const;
340 bool TextTracksVisible() const;
342 static void SetTextTrackKindUserPreferenceForAllMediaElements(Document*);
343 void AutomaticTrackSelectionForUpdatedUserPreference();
345 // Returns the MediaControls, or null if they have not been added yet.
346 // Note that this can be non-null even if there is no controls attribute.
347 MediaControls* GetMediaControls() const;
349 // Notifies the media element that the media controls became visible, so
350 // that text track layout may be updated to avoid overlapping them.
351 void MediaControlsDidBecomeVisible();
353 void SourceWasRemoved(HTMLSourceElement*);
354 void SourceWasAdded(HTMLSourceElement*);
356 // ScriptWrappable functions.
357 bool HasPendingActivity() const override;
359 AudioSourceProviderClient* AudioSourceNode() {
360 return audio_source_node_.Get();
362 void SetAudioSourceNode(AudioSourceProviderClient*);
364 AudioSourceProvider& GetAudioSourceProvider() {
365 return audio_source_provider_;
368 enum InvalidURLAction { kDoNothing, kComplain };
369 bool IsSafeToLoadURL(const KURL&, InvalidURLAction);
371 // Checks to see if current media data is CORS-same-origin.
372 bool IsMediaDataCorsSameOrigin() const;
374 void ScheduleEvent(Event*);
376 // Returns the "effective media volume" value as specified in the HTML5 spec.
377 double EffectiveMediaVolume() const;
379 // Predicates also used when dispatching wrapper creation (cf.
380 // [SpecialWrapFor] IDL attribute usage.)
381 virtual bool IsHTMLAudioElement() const { return false; }
382 virtual bool IsHTMLVideoElement() const { return false; }
384 void VideoWillBeDrawnToCanvas() const;
386 const WebRemotePlaybackClient* RemotePlaybackClient() const {
387 return remote_playback_client_;
390 const AutoplayPolicy& GetAutoplayPolicy() const { return *autoplay_policy_; }
392 WebMediaPlayer::LoadType GetLoadType() const;
394 #if BUILDFLAG(IS_TIZEN_TV)
395 WebString GetContentMIMEType() override;
396 void NotifyTrackInfoToBrowser(int active_track_id) override;
397 void RequestReload(const KURL& new_url);
398 void SetTranslatedURL(const String&) override;
399 bool IsHTMLMediaElement() const override { return true; }
400 WTF::String GetUrl() const override { return current_src_.GetSource(); }
401 void SetParentalRatingResult(bool is_pass) override;
402 TextTrack* FindTextTrack(int band_type, const std::string& info);
403 void OnInbandTextTrack(const std::string& info,
405 int action) override;
406 void AddInbandCue(const std::string& info,
409 long long int start_time,
410 long long int end_time) override;
411 void SetPreferTextLang(const String& lang) override;
414 #if defined(TIZEN_MULTIMEDIA)
415 void OnLivePlaybackComplete() override;
418 bool HasMediaSource() const { return media_source_attachment_.get(); }
420 // Return true if element is paused and won't resume automatically if it
421 // becomes visible again.
422 bool PausedWhenVisible() const;
424 void DidAudioOutputSinkChanged(const String& hashed_device_id);
426 void SetCcLayerForTesting(cc::Layer* layer) { SetCcLayer(layer); }
428 // This should be called directly after creation.
429 void SetMediaPlayerHostForTesting(
430 mojo::PendingAssociatedRemote<media::mojom::blink::MediaPlayerHost> host);
432 bool IsShowPosterFlagSet() const { return show_poster_flag_; }
434 // What LocalFrame should own our player? Normally, players are tied to their
435 // HTMLMediaElement's LocalFrame for metrics, network fetch, etc. This has
436 // the side-effect of requiring that a player is destroyed when the element's
437 // frame changes. That causes playback to be user-visibly interrupted,
438 // potentially for multiple seconds.
440 // In some very restricted cases, related to picture-in-picture playback, it
441 // is okay to keep the player even when the element is moved to a new
442 // document. It requires that everything is same-origin, and that lifetimes
443 // are looked after carefully so that the player does not outlive the frame
444 // that owns it. However, it permits seamless playback when transitioning to
445 // picture in picture. In this case, this function will return a different
446 // frame than our own.
448 // Note that new players can be created in this frame as well, so that a
449 // transfer back to the original opener frame when picture in picture is
450 // closed can be seamless too, even if the player was recreated for some
451 // reason while in picture in picture mode.
452 LocalFrame* LocalFrameForPlayer();
455 // Assert the correct order of the children in shadow dom when DCHECK is on.
456 static void AssertShadowRootChildren(ShadowRoot&);
458 HTMLMediaElement(const QualifiedName&, Document&);
459 ~HTMLMediaElement() override;
462 // Returns a constant reference to the HeapMojoAssociatedRemoteSet holding all
463 // the bound remotes for the media::mojom::blink::MediaPlayerObserver
464 // interface. Needed to allow sending messages directly from
465 // HTMLMediaElement's subclasses.
466 const HeapMojoAssociatedRemoteSet<media::mojom::blink::MediaPlayerObserver>&
467 GetMediaPlayerObserverRemoteSet() {
468 return media_player_observer_remote_set_->Value();
471 void ParseAttribute(const AttributeModificationParams&) override;
472 void FinishParsingChildren() final;
473 bool IsURLAttribute(const Attribute&) const override;
474 void AttachLayoutTree(AttachContext&) override;
475 void ParserDidSetAttributes() override;
476 void CloneNonAttributePropertiesFrom(const Element&,
477 NodeCloningData&) override;
479 InsertionNotificationRequest InsertedInto(ContainerNode&) override;
480 void RemovedFrom(ContainerNode&) override;
482 // Return true if media is cross origin from the current document
483 // and has not passed a cors check, meaning that we should return
484 // as little information as possible about it.
485 bool MediaShouldBeOpaque() const;
487 void DidMoveToNewDocument(Document& old_document) override;
488 virtual KURL PosterImageURL() const { return KURL(); }
490 // Called after the creation of |web_media_player_|.
491 virtual void OnWebMediaPlayerCreated() {}
492 virtual void OnWebMediaPlayerCleared() {}
494 void UpdateLayoutObject();
497 // Friend class for testing.
498 friend class ContextMenuControllerTest;
499 friend class HTMLMediaElementTest;
500 friend class PictureInPictureControllerTestWithWidget;
501 friend class VideoWakeLockTest;
503 class SourceMetadata {
507 enum class SourceVisibility { kVisibleToApp, kInvisibleToApp };
508 SourceMetadata() = default;
509 void SetSource(const KURL& src, SourceVisibility visibility) {
511 invisible_to_app_ = visibility == SourceVisibility::kInvisibleToApp;
513 const KURL& GetSourceIfVisible() const {
514 return invisible_to_app_ ? NullURL() : src_;
516 const KURL& GetSource() const { return src_; }
521 // If true, then |current_src| is used only for internal loading and safety
522 // checks, and for logging that is not visible to apps, either. For example,
523 // when loading from a MediaSourceHandle as srcObject, this would be true.
524 bool invisible_to_app_ = false;
527 bool HasPendingActivityInternal() const;
529 void ResetMediaPlayerAndMediaSource();
531 bool AlwaysCreateUserAgentShadowRoot() const final { return true; }
532 bool AreAuthorShadowsAllowed() const final { return false; }
534 bool SupportsFocus() const final;
536 bool disallow_layout_updates_for_accessibility_only = false) const final;
537 bool IsKeyboardFocusable() const final;
538 int DefaultTabIndex() const final;
539 bool LayoutObjectIsNeeded(const DisplayStyle&) const override;
540 LayoutObject* CreateLayoutObject(const ComputedStyle&) override;
541 void DidNotifySubtreeInsertionsToDocument() override;
542 void DidRecalcStyle(const StyleRecalcChange) final;
544 bool CanStartSelection() const override { return false; }
546 bool IsInteractiveContent() const final;
548 // ExecutionContextLifecycleStateObserver functions.
549 void ContextLifecycleStateChanged(mojom::FrameLifecycleState) override;
550 void ContextDestroyed() override;
552 virtual void OnPlay() {}
553 virtual void OnLoadStarted() {}
554 virtual void OnLoadFinished() {}
556 // Handles playing of media element when audio descriptions are finished
558 void OnSpeakingCompleted();
560 void SetShowPosterFlag(bool value);
562 void SetReadyState(ReadyState);
563 void SetNetworkState(WebMediaPlayer::NetworkState);
565 // WebMediaPlayerClient implementation.
566 void NetworkStateChanged() final;
567 void ReadyStateChanged() final;
568 void TimeChanged() final;
569 void Repaint() final;
570 void DurationChanged() final;
571 void SizeChanged() final;
572 void OnFirstFrame(base::TimeTicks frame_time,
573 size_t bytes_to_first_frame) override {}
575 void SetCcLayer(cc::Layer*) final;
576 WebMediaPlayer::TrackId AddAudioTrack(const WebString&,
577 WebMediaPlayerClient::AudioTrackKind,
581 void RemoveAudioTrack(WebMediaPlayer::TrackId) final;
582 WebMediaPlayer::TrackId AddVideoTrack(const WebString&,
583 WebMediaPlayerClient::VideoTrackKind,
587 void RemoveVideoTrack(WebMediaPlayer::TrackId) final;
588 #if BUILDFLAG(IS_TIZEN_TV)
589 void AddTextTrack(const WebString& id,
590 const WebString& kind,
591 const WebString& label,
592 const WebString& language) final;
594 void MediaSourceOpened(WebMediaSource*) final;
595 void RemotePlaybackCompatibilityChanged(const WebURL&,
596 bool is_compatible) final;
597 bool HasSelectedVideoTrack() final;
598 WebMediaPlayer::TrackId GetSelectedVideoTrackId() final;
599 bool WasAlwaysMuted() final;
600 bool HasNativeControls() final;
601 bool IsAudioElement() final;
602 DisplayType GetDisplayType() const override;
603 WebRemotePlaybackClient* RemotePlaybackClient() final {
604 return remote_playback_client_;
606 gfx::ColorSpace TargetColorSpace() override;
607 bool WasAutoplayInitiated() override;
608 bool IsInAutoPIP() const override { return false; }
609 void ResumePlayback() final;
610 void PausePlayback(PauseReason) final;
611 void DidPlayerStartPlaying() override;
612 void DidPlayerPaused(bool stream_ended) override;
613 void DidPlayerMutedStatusChange(bool muted) override;
614 void DidMediaMetadataChange(bool has_audio,
616 media::AudioCodec audio_codec,
617 media::VideoCodec video_codec,
618 media::MediaContentType media_content_type,
619 bool is_encrypted_media) override;
620 void DidPlayerMediaPositionStateChange(double playback_rate,
621 base::TimeDelta duration,
622 base::TimeDelta position,
623 bool end_of_media) override;
624 void DidDisableAudioOutputSinkChanges() override;
625 void DidUseAudioServiceChange(bool uses_audio_service) override;
626 void DidPlayerSizeChange(const gfx::Size& size) override;
627 void OnRemotePlaybackDisabled(bool disabled) override;
628 #if defined(TIZEN_MULTIMEDIA)
629 void SuspendPlayer() override;
632 // Returns a reference to the mojo remote for the MediaPlayerHost interface,
633 // requesting it first from the BrowserInterfaceBroker if needed. It is an
634 // error to call this method before having access to the document's frame.
635 media::mojom::blink::MediaPlayerHost& GetMediaPlayerHostRemote();
637 // media::mojom::MediaPlayer implementation.
638 void RequestPlay() override;
639 void RequestPause(bool triggered_by_user) override;
640 void RequestSeekForward(base::TimeDelta seek_time) override;
641 void RequestSeekBackward(base::TimeDelta seek_time) override;
642 void RequestSeekTo(base::TimeDelta seek_time) override;
643 void RequestEnterPictureInPicture() override {}
644 void RequestMute(bool mute) override;
645 void SetVolumeMultiplier(double multiplier) override;
646 void SetPersistentState(bool persistent) override {}
647 void SetPowerExperimentState(bool enabled) override;
648 void SetAudioSinkId(const String&) override;
649 void SuspendForFrameClosed() override;
650 void RequestMediaRemoting() override {}
652 #if defined(TIZEN_MULTIMEDIA_SUPPORT)
653 void MediaPlayerHidden() final;
654 void MediaPlayerShown() final;
657 void NotifyPlayingUrl();
660 #if BUILDFLAG(IS_TIZEN_TV)
661 void DisableOtherAudioTracks(AudioTrack*);
662 int FindMatchTrack(const String& lang);
666 void LoadTimerFired(TimerBase*);
667 void ProgressEventTimerFired();
668 void PlaybackProgressTimerFired();
669 void ScheduleTimeupdateEvent(bool periodic_event);
670 void StartPlaybackProgressTimer();
671 void StartProgressEventTimer();
672 void StopPeriodicTimers();
674 void Seek(double time);
676 void AddPlayedRange(double start, double end);
678 // FIXME: Rename to scheduleNamedEvent for clarity.
679 void ScheduleEvent(const AtomicString& event_name);
682 void InvokeLoadAlgorithm();
683 void InvokeResourceSelectionAlgorithm();
685 void SelectMediaResource();
686 void LoadResource(const WebMediaPlayerSource&, const String& content_type);
687 void StartPlayerLoad();
688 void SetPlayerPreload();
689 void ScheduleNextSourceChild();
690 void LoadSourceFromObject();
691 void LoadSourceFromAttribute();
692 void LoadNextSourceChild();
693 void ClearMediaPlayer();
694 void ClearMediaPlayerAndAudioSourceProviderClientWithoutLocking();
695 bool HavePotentialSourceChild();
696 void NoneSupported(const String&);
697 void MediaEngineError(MediaError*);
698 void CancelPendingEventsAndCallbacks();
699 void WaitForSourceChange();
700 void SetIgnorePreloadNone();
702 KURL SelectNextSourceChild(String* content_type, InvalidURLAction);
704 void MediaLoadingFailed(WebMediaPlayer::NetworkState, const String&);
706 // deferred loading (preload=none)
707 bool LoadIsDeferred() const;
709 void CancelDeferredLoad();
710 void StartDeferredLoad();
711 void ExecuteDeferredLoad();
712 void DeferredLoadTimerFired(TimerBase*);
714 void MarkCaptionAndSubtitleTracksAsUnconfigured();
716 // This does not check user gesture restrictions.
719 // This does not stop autoplay visibility observation.
720 // By default, will pause the video and speech.
721 void PauseInternal(PlayPromiseError code, bool pause_speech = true);
723 // By default, will pause the video and speech.
724 void UpdatePlayState(bool pause_speech = true);
726 bool PotentiallyPlaying() const;
727 bool StoppedDueToErrors() const;
728 bool CouldPlayIfEnoughData() const override;
730 // Generally the presence of the loop attribute should be considered to mean
731 // playback has not "ended", as "ended" and "looping" are mutually exclusive.
733 // https://html.spec.whatwg.org/C/#ended-playback
734 enum class LoopCondition { kIncluded, kIgnored };
735 bool EndedPlayback(LoopCondition = LoopCondition::kIncluded) const;
737 void SetShouldDelayLoadEvent(bool);
739 double EarliestPossiblePosition() const;
740 double CurrentPlaybackPosition() const;
741 double OfficialPlaybackPosition() const;
742 void SetOfficialPlaybackPosition(double) const;
743 void RequireOfficialPlaybackPositionUpdate() const;
745 void EnsureMediaControls();
746 void UpdateControlsVisibility();
748 TextTrackContainer& EnsureTextTrackContainer();
750 void ChangeNetworkStateFromLoadingToIdle();
752 WebMediaPlayer::CorsMode CorsMode() const;
754 // Returns the "direction of playback" value as specified in the HTML5 spec.
755 enum DirectionOfPlayback { kBackward, kForward };
756 DirectionOfPlayback GetDirectionOfPlayback() const;
758 // Creates placeholder AudioTrack and/or VideoTrack objects when
759 // WebMediaPlayer objects advertise they have audio and/or video, but don't
760 // explicitly signal them via addAudioTrack() and addVideoTrack().
761 // FIXME: Remove this once all WebMediaPlayer implementations properly report
763 void CreatePlaceholderTracksIfNecessary();
765 void SetNetworkState(NetworkState, bool update_media_controls = true);
767 void AudioTracksTimerFired(TimerBase*);
769 void ScheduleResolvePlayPromises();
770 void ScheduleRejectPlayPromises(PlayPromiseError);
771 void ScheduleNotifyPlaying();
772 void ResolveScheduledPlayPromises();
773 void RejectScheduledPlayPromises();
774 void RejectPlayPromises(DOMExceptionCode, const String&);
775 void RejectPlayPromisesInternal(DOMExceptionCode, const String&);
777 void OnRemovedFromDocumentTimerFired(TimerBase*);
779 void SetError(MediaError* error);
780 void ReportCurrentTimeToMediaSource();
782 void ResetMojoState();
783 void OnRemotePlaybackMetadataChange();
785 // Determine if we should reuse the player when moving the element from
786 // |old_document| to |new_document|
787 bool ShouldReusePlayer(Document& old_document, Document& new_document) const;
789 // Adds a new MediaPlayerObserver remote that will be notified about media
790 // player events and returns a receiver that an observer implementation can
792 mojo::PendingAssociatedReceiver<media::mojom::blink::MediaPlayerObserver>
793 AddMediaPlayerObserverAndPassReceiver();
795 // Timers used to schedule one-shot tasks with no delay.
796 HeapTaskRunnerTimer<HTMLMediaElement> load_timer_;
797 HeapTaskRunnerTimer<HTMLMediaElement> audio_tracks_timer_;
798 HeapTaskRunnerTimer<HTMLMediaElement> removed_from_document_timer_;
799 // Use a low precision timer for repeating tasks to avoid excessive Idle Wake
800 // Up frequency, especially when WebRTC is used and the page contains many
801 // HTMLMediaElements.
802 LowPrecisionTimer progress_event_timer_;
803 LowPrecisionTimer playback_progress_timer_;
805 Member<TimeRanges> played_time_ranges_;
806 Member<EventQueue> async_event_queue_;
808 double playback_rate_;
809 double default_playback_rate_;
810 NetworkState network_state_;
811 ReadyState ready_state_;
812 ReadyState ready_state_maximum_;
814 SourceMetadata current_src_;
815 KURL current_src_after_redirects_;
817 Member<MediaStreamDescriptor> src_object_stream_descriptor_;
818 Member<MediaSourceHandle> src_object_media_source_handle_;
820 // To prevent potential regression when extended by the MSE API, do not set
821 // |error_| outside of constructor and SetError().
822 Member<MediaError> error_;
825 double last_seek_time_;
827 absl::optional<base::ElapsedTimer> previous_progress_time_;
829 // Cached duration to suppress duplicate events if duration unchanged.
832 // The last time a timeupdate event was sent in movie time.
833 double last_time_update_event_media_time_;
835 // The default playback start position.
836 double default_playback_start_position_;
841 kLoadingFromSrcObject,
843 kLoadingFromSourceElement
845 LoadState load_state_;
846 Member<HTMLSourceElement> current_source_node_;
847 Member<Node> next_child_node_to_consider_;
849 // "Deferred loading" state (for preload=none).
850 enum DeferredLoadState {
851 // The load is not deferred.
853 // The load is deferred, and waiting for the task to set the
854 // delaying-the-load-event flag (to false).
855 kWaitingForStopDelayingLoadEventTask,
856 // The load is the deferred, and waiting for a triggering event.
858 // The load is deferred, and waiting for the task to set the
859 // delaying-the-load-event flag, after which the load will be executed.
860 kExecuteOnStopDelayingLoadEventTask
862 DeferredLoadState deferred_load_state_;
863 HeapTaskRunnerTimer<HTMLMediaElement> deferred_load_timer_;
865 std::unique_ptr<WebMediaPlayer> web_media_player_;
866 cc::Layer* cc_layer_;
868 // These two fields must be carefully set and reset: the actual derived type
869 // of the attachment (same-thread vs cross-thread, for instance) must be the
870 // same semantic as the actual derived type of the tracer. Further, if there
871 // is no attachment, then there must be no tracer that's tracking an active
872 // attachment. Note that some kinds of attachments do not require a tracer;
873 // see MediaSourceAttachment::StartAttachingToMediaElement() for details.
874 scoped_refptr<MediaSourceAttachment> media_source_attachment_;
875 Member<MediaSourceTracer> media_source_tracer_;
877 // Stores "official playback position", updated periodically from "current
878 // playback position". Official playback position should not change while
879 // scripts are running. See setOfficialPlaybackPosition().
880 mutable double official_playback_position_;
881 mutable bool official_playback_position_needs_update_;
883 double fragment_end_time_;
885 typedef unsigned PendingActionFlags;
886 PendingActionFlags pending_action_flags_;
888 // FIXME: HTMLMediaElement has way too many state bits.
890 bool should_delay_load_event_ : 1;
891 bool have_fired_loaded_data_ : 1;
892 bool can_autoplay_ : 1;
896 bool paused_by_context_paused_ : 1;
897 bool show_poster_flag_ : 1;
899 // data has not been loaded since sending a "stalled" event
900 bool sent_stalled_event_ : 1;
902 bool ignore_preload_none_ : 1;
904 bool text_tracks_visible_ : 1;
905 bool should_perform_automatic_track_selection_ : 1;
907 bool tracks_are_ready_ : 1;
908 bool processing_preference_change_ : 1;
910 bool was_always_muted_ : 1;
912 #if defined(TIZEN_MULTIMEDIA)
913 bool live_playback_complete_ : 1;
916 #if BUILDFLAG(IS_TIZEN_TV)
917 bool text_track_menu_on_ : 1;
918 int active_text_track_id_{-1};
919 String content_mime_type_;
920 bool is_translated_url_ : 1;
921 std::string prefer_text_lang_{"und"}; // only for ts
923 // Set if the user has used the context menu to set the visibility of the
925 absl::optional<bool> user_wants_controls_visible_;
927 // Whether or not |web_media_player_| should apply pitch adjustments at
928 // playback raters other than 1.0.
929 bool preserves_pitch_ = true;
931 #if BUILDFLAG(IS_TIZEN_TV)
932 bool is_deactivate_ : 1;
935 #if defined(TIZEN_MULTIMEDIA_SUPPORT)
936 // This is to suppress stuff set to JS. Do not use it for other purposes.
937 bool suppress_events_ : 1;
940 // Whether the player disables the Remote Playback feature.
941 bool is_remote_playback_disabled_ = false;
942 // Whether the player is rendering remotely.
943 bool is_remote_rendering_ = false;
944 // Whether the media content is encrypted.
945 bool is_encrypted_media_ = false;
946 WebString remote_device_friendly_name_;
947 media::AudioCodec audio_codec_ = media::AudioCodec::kUnknown;
948 media::VideoCodec video_codec_ = media::VideoCodec::kUnknown;
950 Member<AudioTrackList> audio_tracks_;
951 Member<VideoTrackList> video_tracks_;
952 Member<TextTrackList> text_tracks_;
953 HeapVector<Member<TextTrack>> text_tracks_when_resource_selection_began_;
955 Member<CueTimeline> cue_timeline_;
957 HeapVector<Member<ScriptPromiseResolver>> play_promise_resolvers_;
958 TaskHandle play_promise_resolve_task_handle_;
959 TaskHandle play_promise_reject_task_handle_;
960 HeapVector<Member<ScriptPromiseResolver>> play_promise_resolve_list_;
961 HeapVector<Member<ScriptPromiseResolver>> play_promise_reject_list_;
962 PlayPromiseError play_promise_error_code_;
964 // HTMLMediaElement and its MediaElementAudioSourceNode in case it is provided
966 Member<AudioSourceProviderClient> audio_source_node_;
968 // Controls browser vocalization within the media element (e.g. to speak cues,
969 // to pause utterance).
970 Member<SpeechSynthesisBase> speech_synthesis_;
972 // AudioClientImpl wraps an AudioSourceProviderClient.
973 // When the audio format is known, Chromium calls setFormat().
974 class AudioClientImpl final : public GarbageCollected<AudioClientImpl>,
975 public WebAudioSourceProviderClient {
977 explicit AudioClientImpl(AudioSourceProviderClient* client)
980 ~AudioClientImpl() override = default;
982 // WebAudioSourceProviderClient
983 void SetFormat(uint32_t number_of_channels, float sample_rate) override;
985 void Trace(Visitor*) const;
988 Member<AudioSourceProviderClient> client_;
991 // AudioSourceProviderImpl wraps a WebAudioSourceProvider.
992 // provideInput() calls into Chromium to get a rendered audio stream.
993 class AudioSourceProviderImpl final : public AudioSourceProvider {
997 AudioSourceProviderImpl() = default;
998 ~AudioSourceProviderImpl() override = default;
1000 // Wraps the given WebAudioSourceProvider.
1001 void Wrap(scoped_refptr<WebAudioSourceProviderImpl>);
1003 // AudioSourceProvider
1004 void SetClient(AudioSourceProviderClient*) override;
1005 void ProvideInput(AudioBus*, int frames_to_process) override;
1007 void Trace(Visitor*) const;
1010 base::Lock provide_input_lock;
1011 scoped_refptr<WebAudioSourceProviderImpl> web_audio_source_provider_
1012 GUARDED_BY(provide_input_lock);
1013 Member<AudioClientImpl> client_;
1016 AudioSourceProviderImpl audio_source_provider_;
1018 // Notify HTMLMediaElement when a document's ExecutionContext is destroyed.
1019 // It allows us to disconnect from a previous document's frame if we were
1020 // using it to support our WebMediaPlayer rather than our current frame.
1021 class OpenerContextObserver final
1022 : public GarbageCollected<OpenerContextObserver>,
1023 public ContextLifecycleObserver {
1025 // Notify `element` when our context is destroyed.
1026 explicit OpenerContextObserver(HTMLMediaElement* element);
1027 ~OpenerContextObserver() final;
1029 void Trace(Visitor* visitor) const final;
1032 void ContextDestroyed() final;
1034 Member<HTMLMediaElement> element_;
1037 // Clean up things that are tied to any previous frame, including the player
1038 // and mojo interfaces, when we switch to a new frame.
1039 void AttachToNewFrame();
1041 Member<Document> opener_document_;
1042 Member<OpenerContextObserver> opener_context_observer_;
1044 friend class AutoplayPolicy;
1045 friend class AutoplayUmaHelperTest;
1046 friend class Internals;
1047 friend class TrackDisplayUpdateScope;
1048 friend class MediaControlsImplTest;
1049 friend class HTMLMediaElementTest;
1050 friend class HTMLMediaElementEventListenersTest;
1051 friend class HTMLVideoElement;
1052 friend class MediaControlInputElementTest;
1053 friend class MediaControlsOrientationLockDelegateTest;
1054 friend class MediaControlsRotateToFullscreenDelegateTest;
1055 friend class MediaControlLoadingPanelElementTest;
1056 friend class ContextMenuControllerTest;
1057 friend class HTMLVideoElementTest;
1059 Member<AutoplayPolicy> autoplay_policy_;
1061 WebRemotePlaybackClient* remote_playback_client_;
1063 Member<MediaControls> media_controls_;
1064 Member<HTMLMediaElementControlsList> controls_list_;
1066 Member<IntersectionObserver> lazy_load_intersection_observer_;
1068 Member<DisallowNewWrapper<
1069 HeapMojoAssociatedRemote<media::mojom::blink::MediaPlayerHost>>>
1070 media_player_host_remote_;
1072 // Note: There's only ever one entry in this set.
1073 Member<DisallowNewWrapper<
1074 HeapMojoAssociatedRemoteSet<media::mojom::blink::MediaPlayerObserver>>>
1075 media_player_observer_remote_set_;
1077 // A receiver set is needed here as there will be different objects in the
1078 // browser communicating with this object. This is done this way to avoid
1079 // routing everything through a single class (e.g. RFHI) and to keep this
1080 // logic contained inside MediaPlayer-related classes.
1081 Member<DisallowNewWrapper<
1082 HeapMojoAssociatedReceiverSet<media::mojom::blink::MediaPlayer,
1084 media_player_receiver_set_;
1088 inline bool IsElementOfType<const HTMLMediaElement>(const Node& node) {
1089 return IsA<HTMLMediaElement>(node);
1092 struct DowncastTraits<HTMLMediaElement> {
1093 static bool AllowFrom(const Node& node) {
1094 auto* html_element = DynamicTo<HTMLElement>(node);
1095 return html_element && AllowFrom(*html_element);
1097 static bool AllowFrom(const HTMLElement& html_element) {
1098 return IsA<HTMLAudioElement>(html_element) ||
1099 IsA<HTMLVideoElement>(html_element);
1103 } // namespace blink
1105 #endif // THIRD_PARTY_BLINK_RENDERER_CORE_HTML_MEDIA_HTML_MEDIA_ELEMENT_H_