2 * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #ifndef HTMLMediaElement_h
27 #define HTMLMediaElement_h
29 #include "core/dom/ActiveDOMObject.h"
30 #include "core/events/GenericEventQueue.h"
31 #include "core/html/HTMLElement.h"
32 #include "core/html/track/TextTrack.h"
33 #include "core/html/track/TextTrackCue.h"
34 #include "core/html/track/vtt/VTTCue.h"
35 #include "platform/PODIntervalTree.h"
36 #include "platform/Supplementable.h"
37 #include "platform/graphics/media/MediaPlayer.h"
38 #include "public/platform/WebMediaPlayerClient.h"
39 #include "public/platform/WebMimeRegistry.h"
42 class WebInbandTextTrack;
49 class AudioSourceProvider;
50 class AudioSourceProviderClient;
56 class HTMLSourceElement;
57 class HTMLTrackElement;
59 class MediaController;
62 class HTMLMediaSource;
68 typedef PODIntervalTree<double, TextTrackCue*> CueIntervalTree;
69 typedef CueIntervalTree::IntervalType CueInterval;
70 typedef Vector<CueInterval> CueList;
72 // FIXME: The inheritance from MediaPlayerClient here should be private inheritance.
73 // But it can't be until the Chromium WebMediaPlayerClientImpl class is fixed so it
74 // no longer depends on typecasting a MediaPlayerClient to an HTMLMediaElement.
76 class HTMLMediaElement : public HTMLElement, public WillBeHeapSupplementable<HTMLMediaElement>, public MediaPlayerClient, public ActiveDOMObject {
77 DEFINE_WRAPPERTYPEINFO();
78 WILL_BE_USING_GARBAGE_COLLECTED_MIXIN(HTMLMediaElement);
80 static blink::WebMimeRegistry::SupportsType supportsType(const ContentType&, const String& keySystem = String());
82 static void setMediaStreamRegistry(URLRegistry*);
83 static bool isMediaStreamURL(const String& url);
85 virtual void trace(Visitor*) override;
87 void clearWeakMembers(Visitor*);
89 blink::WebMediaPlayer* webMediaPlayer() const { return m_player ? m_player->webMediaPlayer() : 0; }
91 virtual bool hasVideo() const { return false; }
92 bool hasAudio() const;
94 bool supportsSave() const;
96 blink::WebLayer* platformLayer() const;
98 enum DelayedActionType {
99 LoadMediaResource = 1 << 0,
100 LoadTextTrackResource = 1 << 1,
101 TextTrackChangesNotification = 1 << 2
103 void scheduleDelayedAction(DelayedActionType);
105 bool isActive() const { return m_active; }
107 bool hasRemoteRoutes() const { return m_remoteRoutesAvailable; }
108 bool isPlayingRemotely() const { return m_playingRemotely; }
111 PassRefPtrWillBeRawPtr<MediaError> error() const;
114 void setSrc(const AtomicString&);
115 const KURL& currentSrc() const { return m_currentSrc; }
117 enum NetworkState { NETWORK_EMPTY, NETWORK_IDLE, NETWORK_LOADING, NETWORK_NO_SOURCE };
118 NetworkState networkState() const;
120 String preload() const;
121 void setPreload(const AtomicString&);
123 PassRefPtrWillBeRawPtr<TimeRanges> buffered() const;
125 String canPlayType(const String& mimeType, const String& keySystem = String()) const;
128 enum ReadyState { HAVE_NOTHING, HAVE_METADATA, HAVE_CURRENT_DATA, HAVE_FUTURE_DATA, HAVE_ENOUGH_DATA };
129 ReadyState readyState() const;
130 bool seeking() const;
133 double currentTime() const;
134 void setCurrentTime(double, ExceptionState&);
135 double duration() const;
137 double defaultPlaybackRate() const;
138 void setDefaultPlaybackRate(double);
139 double playbackRate() const;
140 void setPlaybackRate(double);
141 void updatePlaybackRate();
142 PassRefPtrWillBeRawPtr<TimeRanges> played();
143 PassRefPtrWillBeRawPtr<TimeRanges> seekable() const;
145 bool autoplay() const;
150 void requestRemotePlayback();
151 void requestRemotePlaybackControl();
154 unsigned webkitAudioDecodedByteCount() const;
155 unsigned webkitVideoDecodedByteCount() const;
157 // media source extensions
158 void closeMediaSource();
159 void durationChanged(double duration, bool requestSeek);
162 bool shouldShowControls() const;
163 double volume() const;
164 void setVolume(double, ExceptionState&);
168 // play/pause toggling that uses the media controller if present. togglePlayStateWillPlay() is
169 // true if togglePlayState() will call play() or unpause() on the media element or controller.
170 bool togglePlayStateWillPlay() const;
171 void togglePlayState();
173 AudioTrackList& audioTracks();
174 void audioTrackChanged();
176 VideoTrackList& videoTracks();
177 void selectedVideoTrackChanged(blink::WebMediaPlayer::TrackId*);
179 PassRefPtrWillBeRawPtr<TextTrack> addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState&);
180 PassRefPtrWillBeRawPtr<TextTrack> addTextTrack(const AtomicString& kind, const AtomicString& label, ExceptionState& exceptionState) { return addTextTrack(kind, label, emptyAtom, exceptionState); }
181 PassRefPtrWillBeRawPtr<TextTrack> addTextTrack(const AtomicString& kind, ExceptionState& exceptionState) { return addTextTrack(kind, emptyAtom, emptyAtom, exceptionState); }
183 TextTrackList* textTracks();
184 CueList currentlyActiveCues() const { return m_currentlyActiveCues; }
186 void addTextTrack(TextTrack*);
187 void removeTextTrack(TextTrack*);
188 void textTracksChanged();
189 void notifyMediaPlayerOfTextTrackChanges();
191 // Implements the "forget the media element's media-resource-specific tracks" algorithm in the HTML5 spec.
192 void forgetResourceSpecificTracks();
194 void didAddTrackElement(HTMLTrackElement*);
195 void didRemoveTrackElement(HTMLTrackElement*);
197 blink::WebMediaPlayer::TrackId addAudioTrack(const String& id, blink::WebMediaPlayerClient::AudioTrackKind, const AtomicString& label, const AtomicString& language, bool enabled);
198 void removeAudioTrack(blink::WebMediaPlayer::TrackId);
199 blink::WebMediaPlayer::TrackId addVideoTrack(const String& id, blink::WebMediaPlayerClient::VideoTrackKind, const AtomicString& label, const AtomicString& language, bool selected);
200 void removeVideoTrack(blink::WebMediaPlayer::TrackId);
202 virtual void mediaPlayerDidAddTextTrack(blink::WebInbandTextTrack*) override final;
203 virtual void mediaPlayerDidRemoveTextTrack(blink::WebInbandTextTrack*) override final;
204 // FIXME: Remove this when WebMediaPlayerClientImpl::loadInternal does not depend on it.
205 virtual KURL mediaPlayerPosterURL() override { return KURL(); }
210 enum GroupKind { CaptionsAndSubtitles, Description, Chapter, Metadata, Other };
212 explicit TrackGroup(GroupKind kind)
213 : visibleTrack(nullptr)
214 , defaultTrack(nullptr)
220 WillBeHeapVector<RefPtrWillBeMember<TextTrack>> tracks;
221 RefPtrWillBeMember<TextTrack> visibleTrack;
222 RefPtrWillBeMember<TextTrack> defaultTrack;
227 void configureTextTrackGroupForLanguage(const TrackGroup&) const;
228 void configureTextTracks();
229 void configureTextTrackGroup(const TrackGroup&);
231 bool textTracksAreReady() const;
232 enum VisibilityChangeAssumption {
233 AssumeNoVisibleChange,
236 void configureTextTrackDisplay(VisibilityChangeAssumption);
237 void updateTextTrackDisplay();
238 void textTrackReadyStateChanged(TextTrack*);
240 void textTrackKindChanged(TextTrack*);
241 void textTrackModeChanged(TextTrack*);
242 void textTrackAddCues(TextTrack*, const TextTrackCueList*);
243 void textTrackRemoveCues(TextTrack*, const TextTrackCueList*);
244 void textTrackAddCue(TextTrack*, PassRefPtrWillBeRawPtr<TextTrackCue>);
245 void textTrackRemoveCue(TextTrack*, PassRefPtrWillBeRawPtr<TextTrackCue>);
247 // EventTarget function.
248 // Both Node (via HTMLElement) and ActiveDOMObject define this method, which
249 // causes an ambiguity error at compile time. This class's constructor
250 // ensures that both implementations return document, so return the result
251 // of one of them here.
252 using HTMLElement::executionContext;
254 bool hasSingleSecurityOrigin() const { return !m_player || (webMediaPlayer() && webMediaPlayer()->hasSingleSecurityOrigin()); }
256 bool isFullscreen() const;
257 void enterFullscreen();
258 void exitFullscreen();
260 bool hasClosedCaptions() const;
261 bool closedCaptionsVisible() const;
262 void setClosedCaptionsVisible(bool);
264 void remoteRouteAvailabilityChanged(bool);
265 void connectedToRemoteDevice();
266 void disconnectedFromRemoteDevice();
268 MediaControls* mediaControls() const;
270 void sourceWasRemoved(HTMLSourceElement*);
271 void sourceWasAdded(HTMLSourceElement*);
273 // ActiveDOMObject functions.
274 virtual bool hasPendingActivity() const override final;
275 virtual void contextDestroyed() override final;
277 #if ENABLE(WEB_AUDIO)
278 AudioSourceProviderClient* audioSourceNode() { return m_audioSourceNode; }
279 void setAudioSourceNode(AudioSourceProviderClient*);
281 AudioSourceProvider* audioSourceProvider();
284 enum InvalidURLAction { DoNothing, Complain };
285 bool isSafeToLoadURL(const KURL&, InvalidURLAction);
287 MediaController* controller() const;
288 void setController(PassRefPtrWillBeRawPtr<MediaController>); // Resets the MediaGroup and sets the MediaController.
290 void scheduleEvent(PassRefPtrWillBeRawPtr<Event>);
292 // Returns the "effective media volume" value as specified in the HTML5 spec.
293 double effectiveMediaVolume() const;
296 bool isFinalizing() const { return m_isFinalizing; }
298 // Oilpan: finalization of the media element is observable from its
299 // attached MediaSource; it entering a closed state.
301 // Express that by having the MediaSource keep a weak reference
302 // to the media element and signal that it wants to be notified
303 // of destruction if it survives a GC, but the media element
305 void setCloseMediaSourceWhenFinalizing();
308 // Predicates also used when dispatching wrapper creation (cf. [SpecialWrapFor] IDL attribute usage.)
309 virtual bool isHTMLAudioElement() const { return false; }
310 virtual bool isHTMLVideoElement() const { return false; }
313 HTMLMediaElement(const QualifiedName&, Document&);
314 virtual ~HTMLMediaElement();
316 virtual void parseAttribute(const QualifiedName&, const AtomicString&) override;
317 virtual void finishParsingChildren() override final;
318 virtual bool isURLAttribute(const Attribute&) const override;
319 virtual void attach(const AttachContext& = AttachContext()) override;
321 virtual void didMoveToNewDocument(Document& oldDocument) override;
323 enum DisplayMode { Unknown, Poster, PosterWaitingForVideo, Video };
324 DisplayMode displayMode() const { return m_displayMode; }
325 virtual void setDisplayMode(DisplayMode mode) { m_displayMode = mode; }
327 void setControllerInternal(PassRefPtrWillBeRawPtr<MediaController>);
329 bool ignoreTrackDisplayUpdateRequests() const { return m_ignoreTrackDisplayUpdate > 0; }
330 void beginIgnoringTrackDisplayUpdateRequests();
331 void endIgnoringTrackDisplayUpdateRequests();
334 void createMediaPlayer();
336 virtual bool alwaysCreateUserAgentShadowRoot() const override final { return true; }
337 virtual bool areAuthorShadowsAllowed() const override final { return false; }
339 virtual bool supportsFocus() const override final;
340 virtual bool isMouseFocusable() const override final;
341 virtual bool rendererIsNeeded(const RenderStyle&) override;
342 virtual RenderObject* createRenderer(RenderStyle*) override;
343 virtual InsertionNotificationRequest insertedInto(ContainerNode*) override final;
344 virtual void didNotifySubtreeInsertionsToDocument() override;
345 virtual void removedFrom(ContainerNode*) override final;
346 virtual void didRecalcStyle(StyleRecalcChange) override final;
348 virtual void didBecomeFullscreenElement() override final;
349 virtual void willStopBeingFullscreenElement() override final;
350 virtual bool isInteractiveContent() const override final;
351 virtual void defaultEventHandler(Event*) override final;
353 // ActiveDOMObject functions.
354 virtual void stop() override final;
356 virtual void updateDisplayState() { }
358 void setReadyState(ReadyState);
359 void setNetworkState(blink::WebMediaPlayer::NetworkState);
361 virtual void mediaPlayerNetworkStateChanged() override final;
362 virtual void mediaPlayerReadyStateChanged() override final;
363 virtual void mediaPlayerTimeChanged() override final;
364 virtual void mediaPlayerDurationChanged() override final;
365 virtual void mediaPlayerPlaybackStateChanged() override final;
366 virtual void mediaPlayerRequestFullscreen() override final;
367 virtual void mediaPlayerRequestSeek(double) override final;
368 virtual void mediaPlayerRepaint() override final;
369 virtual void mediaPlayerSizeChanged() override final;
370 virtual void mediaPlayerSetWebLayer(blink::WebLayer*) override final;
371 virtual void mediaPlayerMediaSourceOpened(blink::WebMediaSource*) override final;
373 void loadTimerFired(Timer<HTMLMediaElement>*);
374 void progressEventTimerFired(Timer<HTMLMediaElement>*);
375 void playbackProgressTimerFired(Timer<HTMLMediaElement>*);
376 void startPlaybackProgressTimer();
377 void startProgressEventTimer();
378 void stopPeriodicTimers();
380 void seek(double time);
382 void checkIfSeekNeeded();
383 void addPlayedRange(double start, double end);
385 void scheduleTimeupdateEvent(bool periodicEvent);
386 void scheduleEvent(const AtomicString& eventName); // FIXME: Rename to scheduleNamedEvent for clarity.
389 void prepareForLoad();
391 void selectMediaResource();
392 void loadResource(const KURL&, ContentType&, const String& keySystem);
393 void startPlayerLoad();
394 void setPlayerPreload();
395 blink::WebMediaPlayer::LoadType loadType() const;
396 void scheduleNextSourceChild();
397 void loadNextSourceChild();
398 void userCancelledLoad();
399 void clearMediaPlayer(int flags);
400 void clearMediaPlayerAndAudioSourceProviderClientWithoutLocking();
401 bool havePotentialSourceChild();
402 void noneSupported();
403 void mediaEngineError(PassRefPtrWillBeRawPtr<MediaError>);
404 void cancelPendingEventsAndCallbacks();
405 void waitForSourceChange();
406 void prepareToPlay();
408 KURL selectNextSourceChild(ContentType*, String* keySystem, InvalidURLAction);
410 void mediaLoadingFailed(blink::WebMediaPlayer::NetworkState);
412 // deferred loading (preload=none)
413 bool loadIsDeferred() const;
415 void cancelDeferredLoad();
416 void startDeferredLoad();
417 void executeDeferredLoad();
418 void deferredLoadTimerFired(Timer<HTMLMediaElement>*);
420 void updateActiveTextTrackCues(double);
421 HTMLTrackElement* showingTrackWithSameKind(HTMLTrackElement*) const;
423 void markCaptionAndSubtitleTracksAsUnconfigured();
425 // This does not check user gesture restrictions.
428 void gesturelessInitialPlayHalted();
429 void autoplayMediaEncountered();
430 void allowVideoRendering();
433 void updatePlayState();
434 bool potentiallyPlaying() const;
435 bool endedPlayback() const;
436 bool stoppedDueToErrors() const;
437 bool couldPlayIfEnoughData() const;
439 void setShouldDelayLoadEvent(bool);
440 void invalidateCachedTime();
441 void refreshCachedTime() const;
443 bool hasMediaControls() const;
444 bool createMediaControls();
445 void configureMediaControls();
447 virtual void* preDispatchEventHandler(Event*) override final;
449 void changeNetworkStateFromLoadingToIdle();
451 const AtomicString& mediaGroup() const;
452 void setMediaGroup(const AtomicString&);
453 void updateMediaController();
454 bool isBlocked() const;
455 bool isBlockedOnMediaController() const;
456 bool isAutoplaying() const { return m_autoplaying; }
458 blink::WebMediaPlayer::CORSMode corsMode() const;
460 // Returns the "direction of playback" value as specified in the HTML5 spec.
461 enum DirectionOfPlayback { Backward, Forward };
462 DirectionOfPlayback directionOfPlayback() const;
464 // Returns the "effective playback rate" value as specified in the HTML5 spec.
465 double effectivePlaybackRate() const;
467 // Creates placeholder AudioTrack and/or VideoTrack objects when WebMemediaPlayer objects
468 // advertise they have audio and/or video, but don't explicitly signal them via
469 // addAudioTrack() and addVideoTrack().
470 // FIXME: Remove this once all WebMediaPlayer implementations properly report their track info.
471 void createPlaceholderTracksIfNecessary();
473 // Sets the selected/enabled tracks if they aren't set before we initially
474 // transition to HAVE_METADATA.
475 void selectInitialTracksIfNecessary();
477 void audioTracksTimerFired(Timer<HTMLMediaElement>*);
479 Timer<HTMLMediaElement> m_loadTimer;
480 Timer<HTMLMediaElement> m_progressEventTimer;
481 Timer<HTMLMediaElement> m_playbackProgressTimer;
482 Timer<HTMLMediaElement> m_audioTracksTimer;
483 RefPtrWillBeMember<TimeRanges> m_playedTimeRanges;
484 OwnPtrWillBeMember<GenericEventQueue> m_asyncEventQueue;
486 double m_playbackRate;
487 double m_defaultPlaybackRate;
488 NetworkState m_networkState;
489 ReadyState m_readyState;
490 ReadyState m_readyStateMaximum;
493 RefPtrWillBeMember<MediaError> m_error;
496 double m_lastSeekTime;
498 double m_previousProgressTime;
500 // Cached duration to suppress duplicate events if duration unchanged.
503 // The last time a timeupdate event was sent (wall clock).
504 double m_lastTimeUpdateEventWallTime;
506 // The last time a timeupdate event was sent in movie time.
507 double m_lastTimeUpdateEventMovieTime;
509 // The default playback start position.
510 double m_defaultPlaybackStartPosition;
513 enum LoadState { WaitingForSource, LoadingFromSrcAttr, LoadingFromSourceElement };
514 LoadState m_loadState;
515 RefPtrWillBeMember<HTMLSourceElement> m_currentSourceNode;
516 RefPtrWillBeMember<Node> m_nextChildNodeToConsider;
518 // "Deferred loading" state (for preload=none).
519 enum DeferredLoadState {
520 // The load is not deferred.
522 // The load is deferred, and waiting for the task to set the
523 // delaying-the-load-event flag (to false).
524 WaitingForStopDelayingLoadEventTask,
525 // The load is the deferred, and waiting for a triggering event.
527 // The load is deferred, and waiting for the task to set the
528 // delaying-the-load-event flag, after which the load will be executed.
529 ExecuteOnStopDelayingLoadEventTask
531 DeferredLoadState m_deferredLoadState;
532 Timer<HTMLMediaElement> m_deferredLoadTimer;
534 OwnPtr<MediaPlayer> m_player;
535 blink::WebLayer* m_webLayer;
537 MediaPlayer::Preload m_preload;
539 DisplayMode m_displayMode;
541 RefPtrWillBeMember<HTMLMediaSource> m_mediaSource;
543 // Cached time value. Only valid when ready state is HAVE_METADATA or
544 // higher, otherwise the current time is assumed to be zero.
545 mutable double m_cachedTime;
547 double m_fragmentEndTime;
549 typedef unsigned PendingActionFlags;
550 PendingActionFlags m_pendingActionFlags;
552 // FIXME: MediaElement has way too many state bits.
553 bool m_userGestureRequiredForPlay : 1;
555 bool m_shouldDelayLoadEvent : 1;
556 bool m_haveFiredLoadedData : 1;
558 bool m_autoplaying : 1;
563 // data has not been loaded since sending a "stalled" event
564 bool m_sentStalledEvent : 1;
566 // time has not changed since sending an "ended" event
567 bool m_sentEndEvent : 1;
569 bool m_closedCaptionsVisible : 1;
571 bool m_completelyLoaded : 1;
572 bool m_havePreparedToPlay : 1;
573 bool m_delayingLoadForPreloadNone : 1;
575 bool m_tracksAreReady : 1;
576 bool m_haveVisibleTextTrack : 1;
577 bool m_processingPreferenceChange : 1;
578 bool m_remoteRoutesAvailable : 1;
579 bool m_playingRemotely : 1;
581 bool m_isFinalizing : 1;
582 bool m_closeMediaSourceWhenFinalizing : 1;
584 double m_lastTextTrackUpdateTime;
585 bool m_initialPlayWithoutUserGestures : 1;
586 bool m_autoplayMediaCounted : 1;
588 RefPtrWillBeMember<AudioTrackList> m_audioTracks;
589 RefPtrWillBeMember<VideoTrackList> m_videoTracks;
590 RefPtrWillBeMember<TextTrackList> m_textTracks;
591 WillBeHeapVector<RefPtrWillBeMember<TextTrack>> m_textTracksWhenResourceSelectionBegan;
593 CueIntervalTree m_cueTree;
595 CueList m_currentlyActiveCues;
596 int m_ignoreTrackDisplayUpdate;
598 #if ENABLE(WEB_AUDIO)
599 // This is a weak reference, since m_audioSourceNode holds a reference to us.
600 // FIXME: Oilpan: Consider making this a strongly traced pointer with oilpan where strong cycles are not a problem.
601 GC_PLUGIN_IGNORE("http://crbug.com/404577")
602 RawPtrWillBeWeakMember<AudioSourceProviderClient> m_audioSourceNode;
605 friend class MediaController;
606 RefPtrWillBeMember<MediaController> m_mediaController;
608 friend class Internals;
609 friend class TrackDisplayUpdateScope;
611 static URLRegistry* s_mediaStreamRegistry;
615 // Template specializations required by PodIntervalTree in debug mode.
617 struct ValueToString<double> {
618 static String string(const double value)
620 return String::number(value);
625 struct ValueToString<TextTrackCue*> {
626 static String string(TextTrackCue* const& cue)
628 return cue->toString();
633 inline bool isHTMLMediaElement(const HTMLElement& element)
635 return isHTMLAudioElement(element) || isHTMLVideoElement(element);
638 DEFINE_HTMLELEMENT_TYPE_CASTS_WITH_FUNCTION(HTMLMediaElement);
642 #endif // HTMLMediaElement_h