From c7ff29ffd31deae32accd988bfade9618027bd42 Mon Sep 17 00:00:00 2001 From: "commit-queue@webkit.org" Date: Mon, 20 Feb 2012 15:02:09 +0000 Subject: [PATCH] Source/WebCore: Added code to support dispatching of missed cues in case of normal playback and event sorting in case of simultaneous event triggering. -related events cuechange, enter, and exit should be sorted and filtered before dispatching https://bugs.webkit.org/show_bug.cgi?id=72171 Patch by Victor Carbune on 2012-02-20 Reviewed by Eric Carlson. Tests: media/track/track-cues-missed.html media/track/track-cues-sorted-before-dispatch.html * html/HTMLMediaElement.cpp: (WebCore::HTMLMediaElement::HTMLMediaElement): Added initialization code for newly added state variables. (WebCore::trackIndexCompare): Static boolean compare function between the index of two tracks. (WebCore): (WebCore::eventTimeCueCompare): Static boolean compare function between events associated with text track cues. (WebCore::HTMLMediaElement::updateActiveTextTrackCues): Added code to sort the events associated with text track cues before dispatching. Each step from the specification is commented within the code. (WebCore::HTMLMediaElement::finishSeek): Added a boolean variable that is needed within the text track update function, to know whether a seek event has occured before or not. (WebCore::HTMLMediaElement::mediaPlayerTimeChanged): Moved the update call for text tracks at the beginning of the function instead of the end. 'ended' events for video should be dispatched after track specific events. * html/HTMLMediaElement.h: Added variables to keep state information required by the text track update algorithm (last time the algorithm was run, and whether a seeking event has occured) (HTMLMediaElement): * html/LoadableTextTrack.cpp: Refactored fireCueChangeEvent method (WebCore::LoadableTextTrack::fireCueChangeEvent): The method dispatches a synchronous cue change event for the track element. * html/LoadableTextTrack.h: (LoadableTextTrack): * html/TextTrack.cpp: Modified the fireCueChange method, cached track index. (WebCore::TextTrack::TextTrack): (WebCore::TextTrack::trackIndex): Cached the track index. (WebCore): (WebCore::TextTrack::invalidateTrackIndex): Invalidates the track. Used when a new track is added in a TextTrackList instance. * html/TextTrack.h: (TextTrack): (WebCore::TextTrack::fireCueChangeEvent): The fireCueChangeEvent has been changed, as events need to be fired asyncronously. * html/TextTrackCue.cpp: Added internal variables to keep the current index position in the track cue order. This is invalidated when an element is inserted before. (WebCore::TextTrackCue::TextTrackCue): (WebCore::TextTrackCue::cueIndex): Getter for the cueIndex. (WebCore): (WebCore::TextTrackCue::invalidateCueIndex): Invalidates the currently stored cue index. (WebCore::TextTrackCue::dispatchEvent): Event dispatching is done asynchronously now. This should be the only method used for event dispatching. (WebCore::TextTrackCue::setIsActive): The setIsActive method no longer dispatches events, but rather just changes the m_isActive variable. * html/TextTrackCue.h: (TextTrackCue): * html/TextTrackCueList.cpp: (WebCore::TextTrackCueList::getCueIndex): Retrieves the cue index, in the track cue order, of a given cue. (WebCore): (WebCore::TextTrackCueList::add): Modified the add method such that all the next cue indexes are invalidated. (WebCore::TextTrackCueList::invalidateCueIndexes): Invalidates all cue indexes starting with a specific position. * html/TextTrackCueList.h: (TextTrackCueList): * html/track/TextTrackList.cpp: (TextTrackList::getTrackIndex): Retrieves the track index position. (TextTrackList::append): Added method for invalidating the text track index in case of changing the list contents. * html/track/TextTrackList.h: (TextTrackList): LayoutTests: -related events cuechange, enter, and exit should be sorted and filtered before dispatching https://bugs.webkit.org/show_bug.cgi?id=72171 Patch by Victor Carbune on 2012-02-20 Reviewed by Eric Carlson. * media/track/captions-webvtt/missed-cues.vtt: Added. * media/track/captions-webvtt/sorted-dispatch.vtt: Added. * media/track/track-cues-cuechange-expected.txt: This test had to be changed because the synchronous dispatch of the events against the HTMLTrackElement doesn't mean that the text track actually has any active cues at the dispatch moment. * media/track/track-cues-cuechange.html: Changed tests structure to guide the entering and exit events according to the asynchronous dispatch done by TextTrack. * media/track/track-cues-missed-expected.txt: Added. * media/track/track-cues-missed.html: Added. * media/track/track-cues-sorted-before-dispatch-expected.txt: Added. * media/track/track-cues-sorted-before-dispatch.html: Added. git-svn-id: http://svn.webkit.org/repository/webkit/trunk@108241 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- LayoutTests/ChangeLog | 19 ++ .../media/track/captions-webvtt/missed-cues.vtt | 27 +++ .../track/captions-webvtt/sorted-dispatch.vtt | 34 +++ .../media/track/track-cues-cuechange-expected.txt | 20 +- LayoutTests/media/track/track-cues-cuechange.html | 18 +- .../media/track/track-cues-missed-expected.txt | 32 +++ LayoutTests/media/track/track-cues-missed.html | 77 +++++++ .../track-cues-sorted-before-dispatch-expected.txt | 22 ++ .../track/track-cues-sorted-before-dispatch.html | 89 ++++++++ Source/WebCore/ChangeLog | 82 +++++++ Source/WebCore/html/HTMLMediaElement.cpp | 237 ++++++++++++++++++--- Source/WebCore/html/HTMLMediaElement.h | 3 + Source/WebCore/html/LoadableTextTrack.cpp | 5 +- Source/WebCore/html/LoadableTextTrack.h | 6 +- Source/WebCore/html/TextTrack.cpp | 26 ++- Source/WebCore/html/TextTrack.h | 11 +- Source/WebCore/html/TextTrackCue.cpp | 44 ++-- Source/WebCore/html/TextTrackCue.h | 16 +- Source/WebCore/html/TextTrackCueList.cpp | 12 ++ Source/WebCore/html/TextTrackCueList.h | 7 +- Source/WebCore/html/track/TextTrackList.cpp | 23 +- Source/WebCore/html/track/TextTrackList.h | 2 + 22 files changed, 723 insertions(+), 89 deletions(-) create mode 100644 LayoutTests/media/track/captions-webvtt/missed-cues.vtt create mode 100644 LayoutTests/media/track/captions-webvtt/sorted-dispatch.vtt create mode 100644 LayoutTests/media/track/track-cues-missed-expected.txt create mode 100644 LayoutTests/media/track/track-cues-missed.html create mode 100644 LayoutTests/media/track/track-cues-sorted-before-dispatch-expected.txt create mode 100644 LayoutTests/media/track/track-cues-sorted-before-dispatch.html diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog index 85e4ada..8de0aa6 100644 --- a/LayoutTests/ChangeLog +++ b/LayoutTests/ChangeLog @@ -1,3 +1,22 @@ +2012-02-20 Victor Carbune + + -related events cuechange, enter, and exit should be sorted and filtered before dispatching + https://bugs.webkit.org/show_bug.cgi?id=72171 + + Reviewed by Eric Carlson. + + * media/track/captions-webvtt/missed-cues.vtt: Added. + * media/track/captions-webvtt/sorted-dispatch.vtt: Added. + * media/track/track-cues-cuechange-expected.txt: This test had to be changed because + the synchronous dispatch of the events against the HTMLTrackElement doesn't mean that + the text track actually has any active cues at the dispatch moment. + * media/track/track-cues-cuechange.html: Changed tests structure to guide the entering and + exit events according to the asynchronous dispatch done by TextTrack. + * media/track/track-cues-missed-expected.txt: Added. + * media/track/track-cues-missed.html: Added. + * media/track/track-cues-sorted-before-dispatch-expected.txt: Added. + * media/track/track-cues-sorted-before-dispatch.html: Added. + 2012-02-20 Vsevolod Vlasov Unreviewed manual rollout of r107970 which breaks table column widths diff --git a/LayoutTests/media/track/captions-webvtt/missed-cues.vtt b/LayoutTests/media/track/captions-webvtt/missed-cues.vtt new file mode 100644 index 0000000..53dff06 --- /dev/null +++ b/LayoutTests/media/track/captions-webvtt/missed-cues.vtt @@ -0,0 +1,27 @@ +WEBVTT +Events should be triggered for missed (skipped) cues during normal playback. + +1 +00:00:00.000 --> 00:00:01.500 A:start T:20% +Bear is Coming!!!!! +And what kind of a bear it is - just have look. + +2 +00:00:02.000 --> 00:00:02.500 A:start T:20% +I said Bear is coming!!!! + +3 +00:00:05.500 --> 00:00:05.501 A:start T:20% +I said Bear is coming now!!!! + +4 +00:00:05.700 --> 00:00:05.701 A:start T:20% +This is the second missed cue in the test. + +5 +00:00:05.800 --> 00:00:05.801 A:start T:20% +Maybe a third missed cue. + +6 +00:00:05.992 --> 00:00:05.993 A:start T:20% +Fourth missed cue. diff --git a/LayoutTests/media/track/captions-webvtt/sorted-dispatch.vtt b/LayoutTests/media/track/captions-webvtt/sorted-dispatch.vtt new file mode 100644 index 0000000..db925d6 --- /dev/null +++ b/LayoutTests/media/track/captions-webvtt/sorted-dispatch.vtt @@ -0,0 +1,34 @@ +WEBVTT +Enter and exit events should be dispatched in a sorted order according to their times. + +0 +00:00:04.000 --> 00:00:04.500 +Missed cue that should not be considered because of seeking. + +1 +00:00:05.100 --> 00:00:05.800 A:start T:20% +Bear is Coming!!!!! + +2 +00:00:05.100 --> 00:00:05.101 +Missed cue 1 + +3 +00:00:05.100 --> 00:00:05.301 +And what kind of a bear it is - just have look. + +4 +00:00:05.100 --> 00:00:05.101 +Missed Cue 2 + +5 +00:00:05.300 --> 00:00:05.800 A:start T:20% +I said Bear is coming!!!! + +6 +00:00:05.990 --> 00:00:05.993 A:start T:20% +I said Bear is coming now!!!! + +7 +00:00:05.994 --> 00:00:05.998 A:start T:20% +Bear is already here diff --git a/LayoutTests/media/track/track-cues-cuechange-expected.txt b/LayoutTests/media/track/track-cues-cuechange-expected.txt index 4744e15..373b525 100644 --- a/LayoutTests/media/track/track-cues-cuechange-expected.txt +++ b/LayoutTests/media/track/track-cues-cuechange-expected.txt @@ -4,53 +4,35 @@ EVENT(canplaythrough) EXPECTED (testTrack.track.cues.length == '3') OK RUN(video.play()) +EVENT(cuechange) from EVENT(cuechange) from TextTrack Cue entered EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK EXPECTED (currentCue.id == '1') OK EVENT(cuechange) from -Cue entered -EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK -EXPECTED (currentCue.id == '1') OK - EVENT(cuechange) from TextTrack Cue exited EVENT(cuechange) from -Cue exited - EVENT(cuechange) from TextTrack Cue entered EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK EXPECTED (currentCue.id == '2') OK EVENT(cuechange) from -Cue entered -EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK -EXPECTED (currentCue.id == '2') OK - EVENT(cuechange) from TextTrack Cue exited EVENT(cuechange) from -Cue exited - EVENT(cuechange) from TextTrack Cue entered EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK EXPECTED (currentCue.id == '3') OK EVENT(cuechange) from -Cue entered -EXPECTED ([object TextTrackCue] == '[object TextTrackCue]') OK -EXPECTED (currentCue.id == '3') OK - EVENT(cuechange) from TextTrack Cue exited -EVENT(cuechange) from -Cue exited - END OF TEST diff --git a/LayoutTests/media/track/track-cues-cuechange.html b/LayoutTests/media/track/track-cues-cuechange.html index 8d6d0f4..5983693 100644 --- a/LayoutTests/media/track/track-cues-cuechange.html +++ b/LayoutTests/media/track/track-cues-cuechange.html @@ -31,25 +31,21 @@ function cueChangedFromTrackElement() { consoleWrite("EVENT(cuechange) from <track>"); - - currentCueIndex = Math.floor(cueChangeCount/2); - currentCue = event.target.track.cues[currentCueIndex]; - checkCue(); - - ++cueChangeCount; - if (cueChangeCount == testTrack.track.cues.length * 2) - endTest(); } function cueChangedFromTextTrack() { consoleWrite("EVENT(cuechange) from TextTrack"); - - currentCueIndex = Math.floor(cueChangeCount/2); + + currentCueIndex = Math.floor(cueChangeCount / 2); currentCue = event.target.cues[currentCueIndex]; checkCue(); + + ++cueChangeCount; + if (cueChangeCount == testTrack.track.cues.length * 2) + endTest(); } - + function checkCue() { if (cueChangeCount % 2 == 0) { diff --git a/LayoutTests/media/track/track-cues-missed-expected.txt b/LayoutTests/media/track/track-cues-missed-expected.txt new file mode 100644 index 0000000..ee86f66 --- /dev/null +++ b/LayoutTests/media/track/track-cues-missed-expected.txt @@ -0,0 +1,32 @@ +Tests that events are triggered for missed (skipped) cues during normal playback. + +EVENT(canplaythrough) +EXPECTED (testTrack.track.cues.length == '6') OK +RUN(video.play()) +EVENT(enter) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '3') OK +EVENT(exit) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '3') OK +EVENT(enter) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '4') OK +EVENT(exit) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '4') OK +EVENT(enter) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '5') OK +EVENT(exit) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '5') OK +EVENT(enter) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '6') OK +EVENT(exit) +EXPECTED (testTrack.track.cues.getCueById(cueCount) == '[object TextTrackCue]') OK +EXPECTED (currentCue.id == '6') OK +EVENT(ended) +END OF TEST + diff --git a/LayoutTests/media/track/track-cues-missed.html b/LayoutTests/media/track/track-cues-missed.html new file mode 100644 index 0000000..7712750 --- /dev/null +++ b/LayoutTests/media/track/track-cues-missed.html @@ -0,0 +1,77 @@ + + + + + + + + + + +

Tests that events are triggered for missed (skipped) cues during normal playback.

+ + + diff --git a/LayoutTests/media/track/track-cues-sorted-before-dispatch-expected.txt b/LayoutTests/media/track/track-cues-sorted-before-dispatch-expected.txt new file mode 100644 index 0000000..907c88a --- /dev/null +++ b/LayoutTests/media/track/track-cues-sorted-before-dispatch-expected.txt @@ -0,0 +1,22 @@ +Tests that all events events are triggered in chronological order. + +EVENT(canplaythrough) +EXPECTED (testTrack.track.cues.length == '8') OK +RUN(video.play()) +EVENT(ended) +Cue event: enter id: 1 time: 5.1 +Cue event: enter id: 3 time: 5.1 +Cue event: enter id: 2 time: 5.1 +Cue event: enter id: 4 time: 5.1 +Cue event: exit id: 2 time: 5.101 +Cue event: exit id: 4 time: 5.101 +Cue event: enter id: 5 time: 5.3 +Cue event: exit id: 3 time: 5.301 +Cue event: exit id: 1 time: 5.8 +Cue event: exit id: 5 time: 5.8 +Cue event: enter id: 6 time: 5.99 +Cue event: exit id: 6 time: 5.993 +Cue event: enter id: 7 time: 5.994 +Cue event: exit id: 7 time: 5.998 +END OF TEST + diff --git a/LayoutTests/media/track/track-cues-sorted-before-dispatch.html b/LayoutTests/media/track/track-cues-sorted-before-dispatch.html new file mode 100644 index 0000000..d6b0e00 --- /dev/null +++ b/LayoutTests/media/track/track-cues-sorted-before-dispatch.html @@ -0,0 +1,89 @@ + + + + + + + + + + +

Tests that all events events are triggered in chronological order.

+ + + diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog index d9207e1..0abb1ad 100644 --- a/Source/WebCore/ChangeLog +++ b/Source/WebCore/ChangeLog @@ -1,3 +1,85 @@ +2012-02-20 Victor Carbune + + Added code to support dispatching of missed cues in case of normal playback + and event sorting in case of simultaneous event triggering. + + -related events cuechange, enter, and exit should be sorted and filtered before dispatching + https://bugs.webkit.org/show_bug.cgi?id=72171 + + Reviewed by Eric Carlson. + + Tests: media/track/track-cues-missed.html + media/track/track-cues-sorted-before-dispatch.html + + * html/HTMLMediaElement.cpp: + (WebCore::HTMLMediaElement::HTMLMediaElement): Added initialization code + for newly added state variables. + (WebCore::trackIndexCompare): Static boolean compare function between + the index of two tracks. + (WebCore): + (WebCore::eventTimeCueCompare): Static boolean compare function between + events associated with text track cues. + (WebCore::HTMLMediaElement::updateActiveTextTrackCues): Added code to + sort the events associated with text track cues before dispatching. + Each step from the specification is commented within the code. + (WebCore::HTMLMediaElement::finishSeek): Added a boolean variable that + is needed within the text track update function, to know whether a seek + event has occured before or not. + (WebCore::HTMLMediaElement::mediaPlayerTimeChanged): Moved the update + call for text tracks at the beginning of the function instead of the end. + 'ended' events for video should be dispatched after track specific events. + * html/HTMLMediaElement.h: Added variables to keep + state information required by the text track update algorithm (last time + the algorithm was run, and whether a seeking event has occured) + (HTMLMediaElement): + + * html/LoadableTextTrack.cpp: Refactored fireCueChangeEvent method + (WebCore::LoadableTextTrack::fireCueChangeEvent): The method dispatches a + synchronous cue change event for the track element. + * html/LoadableTextTrack.h: + (LoadableTextTrack): + * html/TextTrack.cpp: Modified the fireCueChange method, cached track index. + (WebCore::TextTrack::TextTrack): + (WebCore::TextTrack::trackIndex): Cached the track index. + (WebCore): + (WebCore::TextTrack::invalidateTrackIndex): Invalidates the track. Used + when a new track is added in a TextTrackList instance. + * html/TextTrack.h: + (TextTrack): + (WebCore::TextTrack::fireCueChangeEvent): The fireCueChangeEvent has been changed, + as events need to be fired asyncronously. + * html/TextTrackCue.cpp: Added internal variables to keep the current index + position in the track cue order. This is invalidated when an element is + inserted before. + (WebCore::TextTrackCue::TextTrackCue): + (WebCore::TextTrackCue::cueIndex): Getter for the cueIndex. + (WebCore): + (WebCore::TextTrackCue::invalidateCueIndex): Invalidates the currently stored + cue index. + (WebCore::TextTrackCue::dispatchEvent): Event dispatching is done asynchronously + now. This should be the only method used for event dispatching. + (WebCore::TextTrackCue::setIsActive): The setIsActive method no longer dispatches + events, but rather just changes the m_isActive variable. + * html/TextTrackCue.h: + (TextTrackCue): + + * html/TextTrackCueList.cpp: + (WebCore::TextTrackCueList::getCueIndex): Retrieves the cue index, in the track cue + order, of a given cue. + (WebCore): + (WebCore::TextTrackCueList::add): Modified the add method such that all the next cue + indexes are invalidated. + (WebCore::TextTrackCueList::invalidateCueIndexes): Invalidates all cue indexes starting + with a specific position. + * html/TextTrackCueList.h: + (TextTrackCueList): + * html/track/TextTrackList.cpp: + (TextTrackList::getTrackIndex): Retrieves the track index position. + (TextTrackList::append): Added method for invalidating the text track index in case of + changing the list contents. + * html/track/TextTrackList.h: + (TextTrackList): + 2012-02-20 Kenichi Ishibashi [WebSocket] Move WebSocketChannel::FrameData into a separate header file diff --git a/Source/WebCore/html/HTMLMediaElement.cpp b/Source/WebCore/html/HTMLMediaElement.cpp index df09d0e..a4092e1 100644 --- a/Source/WebCore/html/HTMLMediaElement.cpp +++ b/Source/WebCore/html/HTMLMediaElement.cpp @@ -77,6 +77,7 @@ #include #include #include +#include #include #include @@ -236,6 +237,7 @@ HTMLMediaElement::HTMLMediaElement(const QualifiedName& tagName, Document* docum #if ENABLE(VIDEO_TRACK) , m_tracksAreReady(true) , m_haveVisibleTextTrack(false) + , m_lastTextTrackUpdateTime(-1) , m_textTracks(0) , m_ignoreTrackDisplayUpdate(0) #endif @@ -934,38 +936,222 @@ void HTMLMediaElement::loadResource(const KURL& initialURL, ContentType& content } #if ENABLE(VIDEO_TRACK) +static bool trackIndexCompare(TextTrack* a, + TextTrack* b) +{ + return a->trackIndex() - b->trackIndex() < 0; +} + +static bool eventTimeCueCompare(const std::pair& a, + const std::pair& b) +{ + // 12 - Sort the tasks in events in ascending time order (tasks with earlier + // times first). + if (a.first != b.first) + return a.first - b.first < 0; + + // If the cues belong to different text tracks, it doesn't make sense to + // compare the two tracks by the relative cue order, so return the relative + // track order. + if (a.second->track() != b.second->track()) + return trackIndexCompare(a.second->track(), b.second->track()); + + // 12 - Further sort tasks in events that have the same time by the + // relative text track cue order of the text track cues associated + // with these tasks. + return a.second->cueIndex() - b.second->cueIndex() < 0; +} + + void HTMLMediaElement::updateActiveTextTrackCues(float movieTime) { + LOG(Media, "HTMLMediaElement::updateActiveTextTracks"); + + // 4.8.10.8 Playing the media resource + + // If the current playback position changes while the steps are running, + // then the user agent must wait for the steps to complete, and then must + // immediately rerun the steps. if (ignoreTrackDisplayUpdateRequests()) return; - CueList previouslyActiveCues = m_currentlyActiveCues; - bool activeSetChanged = false; + // 1 - Let current cues be a list of cues, initialized to contain all the + // cues of all the hidden, showing, or showing by default text tracks of the + // media element (not the disabled ones) whose start times are less than or + // equal to the current playback position and whose end times are greater + // than the current playback position. + Vector currentCues; + + // The user agent must synchronously unset [the text track cue active] flag + // whenever ... the media element's readyState is changed back to HAVE_NOTHING. + if (m_readyState != HAVE_NOTHING && m_player) + currentCues = m_cueTree.allOverlaps(m_cueTree.createInterval(movieTime, movieTime)); + + Vector affectedCues; + Vector previousCues; + Vector missedCues; + + // 2 - Let other cues be a list of cues, initialized to contain all the cues + // of hidden, showing, and showing by default text tracks of the media + // element that are not present in current cues. + previousCues = m_currentlyActiveCues; + + // 3 - Let last time be the current playback position at the time this + // algorithm was last run for this media element, if this is not the first + // time it has run. + float lastTime = m_lastTextTrackUpdateTime; + + // 4 - If the current playback position has, since the last time this + // algorithm was run, only changed through its usual monotonic increase + // during normal playback, then let missed cues be the list of cues in other + // cues whose start times are greater than or equal to last time and whose + // end times are less than or equal to the current playback position. + // Otherwise, let missed cues be an empty list. + if (lastTime >= 0 && m_lastSeekTime <= lastTime) { + Vector potentiallySkippedCues = + m_cueTree.allOverlaps(m_cueTree.createInterval(lastTime, movieTime)); + + for (size_t i = 0; i < potentiallySkippedCues.size(); ++i) { + float cueStartTime = potentiallySkippedCues[i].low(); + float cueEndTime = potentiallySkippedCues[i].high(); + + if (cueStartTime > lastTime && cueEndTime < movieTime) + missedCues.append(potentiallySkippedCues[i]); + } + } - // The user agent must synchronously unset [the text track cue active] flag whenever ... the media - // element's readyState is changed back to HAVE_NOTHING. - if (m_readyState == HAVE_NOTHING || !m_player) - m_currentlyActiveCues.shrink(0); - else - m_currentlyActiveCues = m_cueTree.allOverlaps(m_cueTree.createInterval(movieTime, movieTime)); - - // FIXME(72171): Events need to be sorted and filtered before dispatching. + m_lastTextTrackUpdateTime = movieTime; + + // 5 - If the time was reached through the usual monotonic increase of the + // current playback position during normal playback, and if the user agent + // has not fired a timeupdate event at the element in the past 15 to 250ms + // and is not still running event handlers for such an event, then the user + // agent must queue a task to fire a simple event named timeupdate at the + // element. (In the other cases, such as explicit seeks, relevant events get + // fired as part of the overall process of changing the current playback + // position.) + if (m_lastSeekTime <= lastTime) + scheduleTimeupdateEvent(false); + + // 6 - If all of the cues in current cues have their text track cue active + // flag set, none of the cues in other cues have their text track cue active + // flag set, and missed cues is empty, then abort these steps. + bool activeSetChanged = missedCues.size(); - for (size_t i = 0; i < previouslyActiveCues.size(); ++i) { - if (!m_currentlyActiveCues.contains(previouslyActiveCues[i])) { - previouslyActiveCues[i].data()->setIsActive(false); + size_t previousCuesSize = previousCues.size(); + for (size_t i = 0; !activeSetChanged && i < previousCuesSize; ++i) + if (!currentCues.contains(previousCues[i]) && previousCues[i].data()->isActive()) activeSetChanged = true; - } - } - for (size_t i = 0; i < m_currentlyActiveCues.size(); ++i) { - if (!previouslyActiveCues.contains(m_currentlyActiveCues[i])) { - m_currentlyActiveCues[i].data()->setIsActive(true); + + size_t currentCuesSize = currentCues.size(); + for (size_t i = 0; !activeSetChanged && i < currentCuesSize; ++i) { + if (!currentCues[i].data()->isActive()) activeSetChanged = true; - } } - - // FIXME(72173): Pause the media element for cues going past their endTime - // during a monotonic time increase. + + if (!activeSetChanged) + return; + + // FIXME(72173): 7 - If the time was reached through the usual monotonic + // increase of the current playback position during normal playback, and + // there are cues in other cues that have their text track cue pause-on-exit + // flag set and that either have their text track cue active flag set or are + // also in missed cues, then immediately pause the media element. + + // 8 - Let events be a list of tasks, initially empty. Each task in this + // list will be associated with a text track, a text track cue, and a time, + // which are used to sort the list before the tasks are queued. + Vector > eventTasks; + + // 8 - Let affected tracks be a list of text tracks, initially empty. + Vector affectedTracks; + + for (size_t i = 0; i < missedCues.size(); ++i) { + // 9 - For each text track cue in missed cues, prepare an event named enter + // for the TextTrackCue object with the text track cue start time. + eventTasks.append(std::make_pair(missedCues[i].data()->startTime(), + missedCues[i].data())); + + // 10 - For each text track in missed cues, prepare an event + // named exit for the TextTrackCue object with the text track cue end + // time. + eventTasks.append(std::make_pair(missedCues[i].data()->endTime(), + missedCues[i].data())); + } + + for (size_t i = 0; i < previousCues.size(); ++i) { + // 10 - For each text track cue in other cues that has its text + // track cue active flag set prepare an event named exit for the + // TextTrackCue object with the text track cue end time. + if (!currentCues.contains(previousCues[i])) + eventTasks.append(std::make_pair(previousCues[i].data()->endTime(), + previousCues[i].data())); + } + + for (size_t i = 0; i < currentCues.size(); ++i) { + // 11 - For each text track cue in current cues that does not have its + // text track cue active flag set, prepare an event named enter for the + // TextTrackCue object with the text track cue start time. + if (!previousCues.contains(currentCues[i])) + eventTasks.append(std::make_pair(currentCues[i].data()->startTime(), + currentCues[i].data())); + } + + // 12 - Sort the tasks in events in ascending time order (tasks with earlier + // times first). + nonCopyingSort(eventTasks.begin(), eventTasks.end(), eventTimeCueCompare); + + for (size_t i = 0; i < eventTasks.size(); ++i) { + if (!affectedTracks.contains(eventTasks[i].second->track())) + affectedTracks.append(eventTasks[i].second->track()); + + // 13 - Queue each task in events, in list order. + RefPtr event; + + // Each event in eventTasks may be either an enterEvent or an exitEvent, + // depending on the time that is associated with the event. This + // correctly identifies the type of the event, since the startTime is + // always less than the endTime. + if (eventTasks[i].first == eventTasks[i].second->startTime()) + event = Event::create(eventNames().enterEvent, false, false); + else + event = Event::create(eventNames().exitEvent, false, false); + + event->setTarget(eventTasks[i].second); + m_asyncEventQueue.enqueueEvent(event.release()); + } + + // 14 - Sort affected tracks in the same order as the text tracks appear in + // the media element's list of text tracks, and remove duplicates. + nonCopyingSort(affectedTracks.begin(), affectedTracks.end(), trackIndexCompare); + + // 15 - For each text track in affected tracks, in the list order, queue a + // task to fire a simple event named cuechange at the TextTrack object, and, + // if the text track has a corresponding track element, to then fire a + // simple event named cuechange at the track element as well. + for (size_t i = 0; i < affectedTracks.size(); ++i) { + RefPtr event = Event::create(eventNames().cuechangeEvent, false, false); + event->setTarget(affectedTracks[i]); + + m_asyncEventQueue.enqueueEvent(event.release()); + + // Fire syncronous cue change event for track elements. + if (affectedTracks[i]->trackType() == TextTrack::TrackElement) + affectedTracks[i]->fireCueChangeEvent(); + } + + // 16 - Set the text track cue active flag of all the cues in the current + // cues, and unset the text track cue active flag of all the cues in the + // other cues. + for (size_t i = 0; i < currentCues.size(); ++i) + currentCues[i].data()->setIsActive(true); + + for (size_t i = 0; i < previousCues.size(); ++i) + if (!currentCues.contains(previousCues[i])) + previousCues[i].data()->setIsActive(false); + + // Update the current active cues. + m_currentlyActiveCues = currentCues; if (activeSetChanged && hasMediaControls()) mediaControls()->updateTextTrackDisplay(); @@ -2696,6 +2882,10 @@ void HTMLMediaElement::mediaPlayerTimeChanged(MediaPlayer*) { LOG(Media, "HTMLMediaElement::mediaPlayerTimeChanged"); +#if ENABLE(VIDEO_TRACK) + updateActiveTextTrackCues(currentTime()); +#endif + beginProcessingMediaPlayerCallback(); invalidateCachedTime(); @@ -2744,9 +2934,6 @@ void HTMLMediaElement::mediaPlayerTimeChanged(MediaPlayer*) m_sentEndEvent = false; updatePlayState(); -#if ENABLE(VIDEO_TRACK) - updateActiveTextTrackCues(now); -#endif endProcessingMediaPlayerCallback(); } diff --git a/Source/WebCore/html/HTMLMediaElement.h b/Source/WebCore/html/HTMLMediaElement.h index 9f047a7..68855ae 100644 --- a/Source/WebCore/html/HTMLMediaElement.h +++ b/Source/WebCore/html/HTMLMediaElement.h @@ -594,10 +594,13 @@ private: #if ENABLE(VIDEO_TRACK) bool m_tracksAreReady : 1; bool m_haveVisibleTextTrack : 1; + float m_lastTextTrackUpdateTime; RefPtr m_textTracks; Vector > m_textTracksWhenResourceSelectionBegan; + CueIntervalTree m_cueTree; + CueList m_currentlyActiveCues; int m_ignoreTrackDisplayUpdate; #endif diff --git a/Source/WebCore/html/LoadableTextTrack.cpp b/Source/WebCore/html/LoadableTextTrack.cpp index b48d8c7..400b42c 100644 --- a/Source/WebCore/html/LoadableTextTrack.cpp +++ b/Source/WebCore/html/LoadableTextTrack.cpp @@ -125,9 +125,10 @@ void LoadableTextTrack::cueLoadingCompleted(TextTrackLoader* loader, bool loadin void LoadableTextTrack::fireCueChangeEvent() { - TextTrack::fireCueChangeEvent(); + RefPtr event = Event::create(eventNames().cuechangeEvent, false, false); ExceptionCode ec = 0; - m_trackElement->dispatchEvent(Event::create(eventNames().cuechangeEvent, false, false), ec); + + m_trackElement->dispatchEvent(event, ec); } size_t LoadableTextTrack::trackElementIndex() diff --git a/Source/WebCore/html/LoadableTextTrack.h b/Source/WebCore/html/LoadableTextTrack.h index b701225..783cdfa 100644 --- a/Source/WebCore/html/LoadableTextTrack.h +++ b/Source/WebCore/html/LoadableTextTrack.h @@ -57,10 +57,12 @@ public: void scheduleLoad(const KURL&); virtual void clearClient(); - + size_t trackElementIndex(); HTMLTrackElement* trackElement() { return m_trackElement; } + virtual void fireCueChangeEvent(); + private: // TextTrackLoaderClient virtual bool shouldLoadCues(TextTrackLoader*) { return true; } @@ -71,8 +73,6 @@ private: LoadableTextTrack(HTMLTrackElement*, const String& kind, const String& label, const String& language, bool isDefault); void loadTimerFired(Timer*); - - virtual void fireCueChangeEvent(); HTMLTrackElement* m_trackElement; Timer m_loadTimer; diff --git a/Source/WebCore/html/TextTrack.cpp b/Source/WebCore/html/TextTrack.cpp index ee8fed9..175171f 100644 --- a/Source/WebCore/html/TextTrack.cpp +++ b/Source/WebCore/html/TextTrack.cpp @@ -37,11 +37,15 @@ #include "Event.h" #include "ExceptionCode.h" +#include "HTMLMediaElement.h" #include "TextTrackCueList.h" +#include "TextTrackList.h" #include "TrackBase.h" namespace WebCore { +static const int invalidTrackIndex = -1; + const AtomicString& TextTrack::subtitlesKeyword() { DEFINE_STATIC_LOCAL(const AtomicString, subtitles, ("subtitles")); @@ -82,6 +86,7 @@ TextTrack::TextTrack(ScriptExecutionContext* context, TextTrackClient* client, c , m_trackType(type) , m_readinessState(NotLoaded) , m_showingByDefault(false) + , m_trackIndex(invalidTrackIndex) { setKind(kind); } @@ -243,12 +248,6 @@ void TextTrack::removeCue(TextTrackCue* cue, ExceptionCode& ec) m_client->textTrackRemoveCue(this, cue); } -void TextTrack::fireCueChangeEvent() -{ - ExceptionCode ec = 0; - dispatchEvent(Event::create(eventNames().cuechangeEvent, false, false), ec); -} - void TextTrack::cueWillChange(TextTrackCue* cue) { if (!m_client) @@ -268,6 +267,21 @@ void TextTrack::cueDidChange(TextTrackCue* cue) m_client->textTrackAddCue(this, cue); } +int TextTrack::trackIndex() +{ + ASSERT(m_mediaElement); + + if (m_trackIndex == invalidTrackIndex) + m_trackIndex = m_mediaElement->textTracks()->getTrackIndex(this); + + return m_trackIndex; +} + +void TextTrack::invalidateTrackIndex() +{ + m_trackIndex = invalidTrackIndex; +} + TextTrackCueList* TextTrack::ensureTextTrackCueList() { if (!m_cues) diff --git a/Source/WebCore/html/TextTrack.h b/Source/WebCore/html/TextTrack.h index ca6db11..9a8eb8b 100644 --- a/Source/WebCore/html/TextTrack.h +++ b/Source/WebCore/html/TextTrack.h @@ -59,7 +59,7 @@ public: return adoptRef(new TextTrack(context, client, kind, label, language, AddTrack)); } virtual ~TextTrack(); - + void setMediaElement(HTMLMediaElement* element) { m_mediaElement = element; } HTMLMediaElement* mediaElement() { return m_mediaElement; } @@ -99,15 +99,19 @@ public: void addCue(PassRefPtr, ExceptionCode&); void removeCue(TextTrackCue*, ExceptionCode&); + virtual void fireCueChangeEvent() { }; + void cueWillChange(TextTrackCue*); void cueDidChange(TextTrackCue*); - - virtual void fireCueChangeEvent(); + DEFINE_ATTRIBUTE_EVENT_LISTENER(cuechange); enum TextTrackType { TrackElement, AddTrack, InBand }; TextTrackType trackType() const { return m_trackType; } + int trackIndex(); + void invalidateTrackIndex(); + protected: TextTrack(ScriptExecutionContext*, TextTrackClient*, const String& kind, const String& label, const String& language, TextTrackType); @@ -124,6 +128,7 @@ private: TextTrackType m_trackType; ReadinessState m_readinessState; bool m_showingByDefault; + int m_trackIndex; }; } // namespace WebCore diff --git a/Source/WebCore/html/TextTrackCue.cpp b/Source/WebCore/html/TextTrackCue.cpp index e1c0126..59c3f3e 100644 --- a/Source/WebCore/html/TextTrackCue.cpp +++ b/Source/WebCore/html/TextTrackCue.cpp @@ -38,11 +38,14 @@ #include "Event.h" #include "DocumentFragment.h" #include "TextTrack.h" +#include "TextTrackCueList.h" #include "WebVTTParser.h" #include namespace WebCore { +static const int invalidCueIndex = -1; + static const AtomicString& startKeyword() { DEFINE_STATIC_LOCAL(const AtomicString, start, ("start")); @@ -87,6 +90,7 @@ TextTrackCue::TextTrackCue(ScriptExecutionContext* context, const String& id, do , m_linePosition(-1) , m_textPosition(50) , m_cueSize(100) + , m_cueIndex(invalidCueIndex) , m_cueAlignment(Middle) , m_scriptExecutionContext(context) , m_isActive(false) @@ -324,6 +328,19 @@ void TextTrackCue::setText(const String& text) cueDidChange(); } +int TextTrackCue::cueIndex() +{ + if (m_cueIndex == invalidCueIndex) + m_cueIndex = track()->cues()->getCueIndex(this); + + return m_cueIndex; +} + +void TextTrackCue::invalidateCueIndex() +{ + m_cueIndex = invalidCueIndex; +} + PassRefPtr TextTrackCue::getCueAsHTML() { if (!m_documentFragment) @@ -337,6 +354,20 @@ void TextTrackCue::setCueHTML(PassRefPtr fragment) m_documentFragment = fragment; } +bool TextTrackCue::dispatchEvent(PassRefPtr event) +{ + // When a TextTrack's mode is disabled: no cues are active, no events fired. + if (!track() || track()->mode() == TextTrack::DISABLED) + return false; + + return EventTarget::dispatchEvent(event); +} + +bool TextTrackCue::dispatchEvent(PassRefPtr event, ExceptionCode &ec) +{ + return EventTarget::dispatchEvent(event, ec); +} + bool TextTrackCue::isActive() { return m_isActive && track() && track()->mode() != TextTrack::DISABLED; @@ -345,19 +376,6 @@ bool TextTrackCue::isActive() void TextTrackCue::setIsActive(bool active) { m_isActive = active; - - // When a TextTrack's mode is disabled: No cues are active, no events are fired ... - if (!track() || track()->mode() == TextTrack::DISABLED) - return; - - ExceptionCode ec = 0; - if (active) - dispatchEvent(Event::create(eventNames().enterEvent, false, false), ec); - else - dispatchEvent(Event::create(eventNames().exitEvent, false, false), ec); - - if (m_track) - m_track->fireCueChangeEvent(); } void TextTrackCue::parseSettings(const String& input) diff --git a/Source/WebCore/html/TextTrackCue.h b/Source/WebCore/html/TextTrackCue.h index 16faf63..b3e0f32 100644 --- a/Source/WebCore/html/TextTrackCue.h +++ b/Source/WebCore/html/TextTrackCue.h @@ -50,7 +50,7 @@ public: { return adoptRef(new TextTrackCue(context, id, start, end, content, settings, pauseOnExit)); } - + enum Direction { Horizontal, VerticalGrowingLeft, VerticalGrowingRight }; enum Alignment { Start, Middle, End }; @@ -92,12 +92,18 @@ public: const String& text() const { return m_content; } void setText(const String&); + int cueIndex(); + void invalidateCueIndex(); + PassRefPtr getCueAsHTML(); void setCueHTML(PassRefPtr); + virtual bool dispatchEvent(PassRefPtr); + bool dispatchEvent(PassRefPtr, ExceptionCode&); + bool isActive(); void setIsActive(bool); - + virtual const AtomicString& interfaceName() const; virtual ScriptExecutionContext* scriptExecutionContext() const; @@ -118,10 +124,10 @@ private: void parseSettings(const String&); void cueWillChange(); void cueDidChange(); - + virtual void refEventTarget() { ref(); } virtual void derefEventTarget() { deref(); } - + String m_id; double m_startTime; double m_endTime; @@ -130,6 +136,8 @@ private: int m_linePosition; int m_textPosition; int m_cueSize; + int m_cueIndex; + Alignment m_cueAlignment; RefPtr m_documentFragment; RefPtr m_track; diff --git a/Source/WebCore/html/TextTrackCueList.cpp b/Source/WebCore/html/TextTrackCueList.cpp index 34b71ca..391b2f6 100644 --- a/Source/WebCore/html/TextTrackCueList.cpp +++ b/Source/WebCore/html/TextTrackCueList.cpp @@ -40,6 +40,11 @@ unsigned long TextTrackCueList::length() const return m_list.size(); } +unsigned long TextTrackCueList::getCueIndex(TextTrackCue* cue) const +{ + return m_list.find(cue); +} + TextTrackCue* TextTrackCueList::item(unsigned index) const { if (index < m_list.size()) @@ -88,6 +93,7 @@ bool TextTrackCueList::add(PassRefPtr prpCue, size_t start, size_t return false; m_list.insert(start, cue); + invalidateCueIndexes(start); return true; } @@ -119,6 +125,12 @@ void TextTrackCueList::clear() m_list.clear(); } +void TextTrackCueList::invalidateCueIndexes(size_t start) +{ + for (size_t i = start; i < m_list.size(); ++i) + m_list[i]->invalidateCueIndex(); +} + } // namespace WebCore #endif diff --git a/Source/WebCore/html/TextTrackCueList.h b/Source/WebCore/html/TextTrackCueList.h index 7a7f904..90b19a5 100644 --- a/Source/WebCore/html/TextTrackCueList.h +++ b/Source/WebCore/html/TextTrackCueList.h @@ -45,6 +45,8 @@ public: ~TextTrackCueList() { } unsigned long length() const; + unsigned long getCueIndex(TextTrackCue*) const; + TextTrackCue* item(unsigned index) const; TextTrackCue* getCueById(const String&) const; TextTrackCueList* activeCues(); @@ -57,10 +59,11 @@ private: TextTrackCueList(); bool add(PassRefPtr, size_t, size_t); void clear(); - + void invalidateCueIndexes(size_t); + Vector > m_list; RefPtr m_activeCues; - + }; } // namespace WebCore diff --git a/Source/WebCore/html/track/TextTrackList.cpp b/Source/WebCore/html/track/TextTrackList.cpp index 77d9673..ead1950 100644 --- a/Source/WebCore/html/track/TextTrackList.cpp +++ b/Source/WebCore/html/track/TextTrackList.cpp @@ -55,6 +55,19 @@ unsigned TextTrackList::length() const return m_addTrackTracks.size() + m_elementTracks.size(); } +unsigned TextTrackList::getTrackIndex(TextTrack *textTrack) +{ + if (textTrack->trackType() == TextTrack::TrackElement) + return static_cast(textTrack)->trackElementIndex(); + + if (textTrack->trackType() == TextTrack::AddTrack) + return m_elementTracks.size() + m_addTrackTracks.find(textTrack); + + ASSERT_NOT_REACHED(); + + return -1; +} + TextTrack* TextTrackList::item(unsigned index) { // 4.8.10.12.1 Text track model @@ -77,13 +90,21 @@ TextTrack* TextTrackList::item(unsigned index) void TextTrackList::append(PassRefPtr prpTrack) { RefPtr track = prpTrack; - + if (track->trackType() == TextTrack::AddTrack) m_addTrackTracks.append(track); else if (track->trackType() == TextTrack::TrackElement) { // Insert tracks added for element in tree order. size_t index = static_cast(track.get())->trackElementIndex(); m_elementTracks.insert(index, track); + + // Invalidate the cached index for all the following tracks. + for (size_t i = index; i < m_elementTracks.size(); ++i) + m_elementTracks[i]->invalidateTrackIndex(); + + for (size_t i = 0; i < m_addTrackTracks.size(); ++i) + m_addTrackTracks[i]->invalidateTrackIndex(); + } else ASSERT_NOT_REACHED(); diff --git a/Source/WebCore/html/track/TextTrackList.h b/Source/WebCore/html/track/TextTrackList.h index 88d9af6..f8f04ff 100644 --- a/Source/WebCore/html/track/TextTrackList.h +++ b/Source/WebCore/html/track/TextTrackList.h @@ -51,6 +51,8 @@ public: ~TextTrackList(); unsigned length() const; + unsigned getTrackIndex(TextTrack*); + TextTrack* item(unsigned index); void append(PassRefPtr); void remove(TextTrack*); -- 2.7.4