1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // SourceBufferStream is a data structure that stores media Buffers in ranges.
6 // Buffers can be appended out of presentation order. Buffers are retrieved by
7 // seeking to the desired start point and calling GetNextBuffer(). Buffers are
8 // returned in sequential order to feed decoder, generally near presentation
9 // order though not necessarily the same as presentation order within GOPs of
10 // out-of-order codecs.
12 #ifndef MEDIA_FILTERS_SOURCE_BUFFER_STREAM_H_
13 #define MEDIA_FILTERS_SOURCE_BUFFER_STREAM_H_
20 #include <type_traits>
24 #include "base/memory/memory_pressure_listener.h"
25 #include "base/memory/raw_ptr.h"
26 #include "base/memory/ref_counted.h"
27 #include "base/time/time.h"
28 #include "media/base/audio_decoder_config.h"
29 #include "media/base/media_export.h"
30 #include "media/base/media_log.h"
31 #include "media/base/ranges.h"
32 #include "media/base/stream_parser_buffer.h"
33 #include "media/base/text_track_config.h"
34 #include "media/base/video_decoder_config.h"
35 #include "media/filters/source_buffer_range.h"
39 // Status returned by GetNextBuffer().
40 // kSuccess: Indicates that the next buffer was returned.
41 // kNeedBuffer: Indicates that we need more data before a buffer can be
43 // kConfigChange: Indicates that the next buffer requires a config change.
44 enum class SourceBufferStreamStatus {
51 enum class SourceBufferStreamType { kAudio, kVideo, kText };
53 // See file-level comment for complete description.
54 class MEDIA_EXPORT SourceBufferStream {
56 using BufferQueue = StreamParser::BufferQueue;
57 using RangeList = std::list<std::unique_ptr<SourceBufferRange>>;
59 // Helper for PrepareRangesForNextAppend and BufferQueueToLogString that
60 // populates |start| and |end| with the presentation interval of |buffers|.
61 static void GetTimestampInterval(const BufferQueue& buffers,
62 base::TimeDelta* start,
63 base::TimeDelta* end);
65 SourceBufferStream(const AudioDecoderConfig& audio_config,
67 SourceBufferStream(const VideoDecoderConfig& video_config,
69 SourceBufferStream(const TextTrackConfig& text_config, MediaLog* media_log);
71 SourceBufferStream(const SourceBufferStream&) = delete;
72 SourceBufferStream& operator=(const SourceBufferStream&) = delete;
74 ~SourceBufferStream();
76 // Signals that the next buffers appended are part of a new coded frame group
77 // starting at |coded_frame_group_start_pts|.
78 void OnStartOfCodedFrameGroup(base::TimeDelta coded_frame_group_start_pts);
80 // Add the |buffers| to the SourceBufferStream. Buffers within the queue are
81 // expected to be in order, but multiple calls to Append() may add buffers out
82 // of order or overlapping. Assumes all buffers within |buffers| are in
83 // presentation order and are non-overlapping.
84 void Append(const BufferQueue& buffers);
86 // Removes buffers between |start| and |end| according to the steps
87 // in the "Coded Frame Removal Algorithm" in the Media Source
89 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-coded-frame-removal
91 // |duration| is the current duration of the presentation. It is
92 // required by the computation outlined in the spec.
93 void Remove(base::TimeDelta start, base::TimeDelta end,
94 base::TimeDelta duration);
96 // Frees up space if the SourceBufferStream is taking up too much memory.
97 // |media_time| is current playback position.
98 bool GarbageCollectIfNeeded(base::TimeDelta media_time, size_t newDataSize);
100 // Gets invoked when the system is experiencing memory pressure, i.e. there's
101 // not enough free memory. The |media_time| is the media playback position at
102 // the time of memory pressure notification (needed for accurate GC). The
103 // |memory_pressure_level| indicates memory pressure severity. The
104 // |force_instant_gc| is used to force the MSE garbage collection algorithm to
105 // be run right away, without waiting for the next append.
106 void OnMemoryPressure(
107 base::TimeDelta media_time,
108 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
109 bool force_instant_gc);
111 // Changes the SourceBufferStream's state so that it will start returning
112 // buffers starting from the closest keyframe before |timestamp|.
113 void Seek(base::TimeDelta timestamp);
115 // Returns true if the SourceBufferStream has seeked to a time without
116 // buffered data and is waiting for more data to be appended.
117 bool IsSeekPending() const;
119 // Notifies the SourceBufferStream that the media duration has been changed to
120 // |duration| so it should drop any data past that point.
121 void OnSetDuration(base::TimeDelta duration);
123 // Fills |out_buffer| with a new buffer. Buffers are presented in order from
124 // the last call to Seek(), or starting with the first buffer appended if
125 // Seek() has not been called yet.
126 // |out_buffer|'s timestamp may be earlier than the |timestamp| passed to
127 // the last Seek() call.
128 // Returns kSuccess if |out_buffer| is filled with a valid buffer, kNeedBuffer
129 // if there is not enough data buffered to fulfill the request, and
130 // kConfigChange if the next buffer requires a config change.
131 SourceBufferStreamStatus GetNextBuffer(
132 scoped_refptr<StreamParserBuffer>* out_buffer);
134 // Returns a list of the buffered time ranges.
135 Ranges<base::TimeDelta> GetBufferedTime() const;
137 // Returns the highest buffered PTS or base::TimeDelta() if nothing is
139 base::TimeDelta GetHighestPresentationTimestamp() const;
141 // Returns the duration of the buffered ranges, which is equivalent
142 // to the end timestamp of the last buffered range. If no data is buffered
143 // then base::TimeDelta() is returned.
144 base::TimeDelta GetBufferedDuration() const;
146 // Returns the size of the buffered data in bytes.
147 size_t GetBufferedSize() const;
149 // Notifies this object that end of stream has been signalled.
150 void MarkEndOfStream();
152 // Clear the end of stream state set by MarkEndOfStream().
153 void UnmarkEndOfStream();
155 const AudioDecoderConfig& GetCurrentAudioDecoderConfig();
156 const VideoDecoderConfig& GetCurrentVideoDecoderConfig();
157 const TextTrackConfig& GetCurrentTextTrackConfig();
159 // Notifies this object that the audio config has changed and buffers in
160 // future Append() calls should be associated with this new config.
161 // If the codec is allowed to change, the caller should set
162 // |allow_codec_change| to true.
163 bool UpdateAudioConfig(const AudioDecoderConfig& config,
164 bool allow_codec_change);
166 // Notifies this object that the video config has changed and buffers in
167 // future Append() calls should be associated with this new config.
168 // If the codec is allowed to change, the caller should set
169 // |allow_codec_change| to true.
170 bool UpdateVideoConfig(const VideoDecoderConfig& config,
171 bool allow_codec_change);
173 // Returns the largest distance between two adjacent buffers in this stream,
174 // or an estimate if no two adjacent buffers have been appended to the stream
176 base::TimeDelta GetMaxInterbufferDistance() const;
178 void set_memory_limit(size_t memory_limit) {
179 memory_limit_ = memory_limit;
182 // A helper function for detecting video/audio config change, so that we
183 // can "peek" the next buffer instead of dequeuing it directly from the source
184 // stream buffer queue.
185 bool IsNextBufferConfigChanged();
188 friend class SourceBufferStreamTest;
190 // Attempts to delete approximately |total_bytes_to_free| amount of data
191 // |ranges_|, starting at the front of |ranges_| and moving linearly forward
192 // through the buffers. Deletes starting from the back if |reverse_direction|
193 // is true. |media_time| is current playback position.
194 // Returns the number of bytes freed.
195 size_t FreeBuffers(size_t total_bytes_to_free,
196 base::TimeDelta media_time,
197 bool reverse_direction);
199 // Attempts to delete approximately |total_bytes_to_free| amount of data from
200 // |ranges_|, starting after the last appended media
201 // (|highest_buffered_end_time_in_append_sequence_|) but before the current
202 // playback position |media_time|.
203 size_t FreeBuffersAfterLastAppended(size_t total_bytes_to_free,
204 base::TimeDelta media_time);
206 // Gets the removal range to secure |total_bytes_to_free| from
207 // [|start_timestamp|, |end_timestamp|).
208 // Returns the size of buffers to secure if future
209 // Remove(|start_timestamp|, |removal_end_timestamp|, duration) is called.
210 // Will not update |removal_end_timestamp| if the returned size is 0.
211 size_t GetRemovalRange(base::TimeDelta start_timestamp,
212 base::TimeDelta end_timestamp,
213 size_t total_bytes_to_free,
214 base::TimeDelta* removal_end_timestamp);
216 // Prepares |range_for_next_append_| so |new_buffers| can be appended.
217 // This involves removing buffers between the end of the previous append
218 // and any buffers covered by the time range in |new_buffers|.
219 // |deleted_buffers| is an output parameter containing candidates for
220 // |track_buffer_| if this method ends up removing the current playback
221 // position from the range.
222 void PrepareRangesForNextAppend(const BufferQueue& new_buffers,
223 BufferQueue* deleted_buffers);
225 // Removes buffers, from the |track_buffer_|, that come after |timestamp|.
226 // Due to out-of-order decode versus presentation times for some kinds of
227 // media, |timestamp| should be the time of a keyframe known by the caller.
228 // |timestamp| must not be kNoTimestamp.
229 void PruneTrackBuffer(const base::TimeDelta timestamp);
231 // Checks to see if |range_with_new_buffers_itr| can be merged with the range
232 // next to it, and merges them if so while preserving correctness of
233 // |range_for_next_append_| and |selected_range_|.
234 void MergeWithNextRangeIfNecessary(
235 const RangeList::iterator& range_with_new_buffers_itr);
237 // Merges any adjacent ranges while preserving correctness of
238 // |range_for_next_append_| and |selected_range_|.
239 void MergeAllAdjacentRanges();
241 // Returns true if |next_gop_timestamp| follows
242 // |highest_timestamp_in_append_sequence_| within fudge room.
243 bool IsNextGopAdjacentToEndOfCurrentAppendSequence(
244 base::TimeDelta next_gop_timestamp) const;
246 // Helper method that returns the timestamp for the next buffer that
247 // |selected_range_| will return from GetNextBuffer() call, or kNoTimestamp
248 // if in between seeking (i.e. |selected_range_| is null).
249 base::TimeDelta GetNextBufferTimestamp();
251 // Finds the range that should contain a coded frame group that begins with
252 // |start_timestamp| (presentation time) and returns the iterator pointing to
253 // it. Returns |ranges_.end()| if there's no such existing range.
254 RangeList::iterator FindExistingRangeFor(base::TimeDelta start_timestamp);
256 // Inserts |new_range| into |ranges_| preserving sorted order. Returns an
257 // iterator in |ranges_| that points to |new_range|. |new_range| becomes owned
259 RangeList::iterator AddToRanges(std::unique_ptr<SourceBufferRange> new_range);
261 // Sets the |selected_range_| to |range| and resets the next buffer position
262 // for the previous |selected_range_|.
263 void SetSelectedRange(SourceBufferRange* range);
265 // Seeks |range| to |seek_timestamp| and then calls SetSelectedRange() with
267 void SeekAndSetSelectedRange(SourceBufferRange* range,
268 base::TimeDelta seek_timestamp);
270 // Resets this stream back to an unseeked state.
271 void ResetSeekState();
273 // Reset state tracking various metadata about the last appended buffer.
274 void ResetLastAppendedState();
276 // Returns true if |seek_timestamp| refers to the beginning of the first range
277 // in |ranges_|, false otherwise or if |ranges_| is empty.
278 bool ShouldSeekToStartOfBuffered(base::TimeDelta seek_timestamp) const;
280 // Returns true if the decode timestamps of |buffers| are monotonically
281 // increasing (within each GOP) since the previous append to the coded frame
282 // group, false otherwise.
283 bool IsDtsMonotonicallyIncreasing(const BufferQueue& buffers);
285 // Returns true if |selected_range_| is the only range in |ranges_| that
286 // HasNextBufferPosition().
287 bool OnlySelectedRangeIsSeeked() const;
289 // Measures the distances between buffer decode timestamps and tracks the max.
290 // This enables a reasonable approximation of adjacency fudge room, even for
291 // out-of-order PTS vs DTS sequences. Returns true if
292 // |max_interbuffer_distance_| was changed.
293 bool UpdateMaxInterbufferDtsDistance(const BufferQueue& buffers);
295 // Sets the config ID for each buffer to |append_config_index_|.
296 void SetConfigIds(const BufferQueue& buffers);
298 // Called to complete a config change. Updates |current_config_index_| to
299 // match the index of the next buffer. Calling this method causes
300 // GetNextBuffer() to stop returning kConfigChange and start returning
302 void CompleteConfigChange();
304 // Sets |selected_range_| and seeks to the nearest keyframe after
305 // |timestamp| if necessary and possible. This method only attempts to
306 // set |selected_range_| if |seleted_range_| is null and |track_buffer_|
308 void SetSelectedRangeIfNeeded(const base::TimeDelta timestamp);
310 // Find a keyframe timestamp that is >= |start_timestamp| and can be used to
311 // find a new selected range.
312 // Returns kNoTimestamp if an appropriate keyframe timestamp could not be
314 base::TimeDelta FindNewSelectedRangeSeekTimestamp(
315 const base::TimeDelta start_timestamp);
317 // Searches |ranges_| for the first keyframe timestamp that is >= |timestamp|.
318 // If |ranges_| doesn't contain a GOP that covers |timestamp| or doesn't
319 // have a keyframe after |timestamp| then kNoTimestamp is returned.
320 base::TimeDelta FindKeyframeAfterTimestamp(const base::TimeDelta timestamp);
322 // Returns "VIDEO" for a video SourceBufferStream, "AUDIO" for an audio
323 // stream, and "TEXT" for a text stream.
324 std::string GetStreamTypeName() const;
326 // (Audio only) If |new_buffers| overlap existing buffers, trims end of
327 // existing buffers to remove overlap. |new_buffers| are not modified.
328 void TrimSpliceOverlap(const BufferQueue& new_buffers);
330 // Returns true if end of stream has been reached, i.e. the
331 // following conditions are met:
332 // 1. end of stream is marked and there is nothing in the track_buffer.
333 // 2. We don't have any ranges, or the last or no range is selected,
334 // or there is a pending seek beyond any existing ranges.
335 bool IsEndOfStreamReached() const;
337 // Deletes the range pointed to by |*itr| and removes it from |ranges_|.
338 // If |*itr| points to |selected_range_|, then |selected_range_| is set to
339 // NULL. After the range is removed, |*itr| is to the range after the one that
340 // was removed or to |ranges_.end()| if the last range was removed.
341 void DeleteAndRemoveRange(RangeList::iterator* itr);
343 // Helper function used when updating |range_for_next_append_|. Returns a
344 // guess of what the next append timestamp will be based on
345 // |last_appended_buffer_timestamp_|, |new_coded_frame_group_| and
346 // |coded_frame_group_start_pts_|. Returns kNoTimestamp if unable to guess,
347 // which can occur prior to first OnStartOfCodedFrameGroup(), or when the most
348 // recent GOP appended to since the last OnStartOfCodedFrameGroup() is
350 base::TimeDelta PotentialNextAppendTimestamp() const;
352 // Helper function used by Remove() and PrepareRangesForNextAppend() to
353 // remove buffers and ranges between |start| and |end|.
354 // |exclude_start| - If set to true, buffers with timestamps that
355 // match |start| are not removed. If set to false, buffers with
356 // timestamps that match |start| will be removed.
357 // |*deleted_buffers| - Filled with buffers for the current playback position
358 // if the removal range included the current playback position. These buffers
359 // can be used as candidates for placing in the |track_buffer_|.
360 void RemoveInternal(base::TimeDelta start,
363 BufferQueue* deleted_buffers);
365 // Helper function used by RemoveInternal() to evaluate whether remove will
366 // disrupt the last appended GOP. If disruption is expected, reset state
367 // tracking the last append. This will trigger frame filtering in Append()
368 // until a new key frame is provided.
369 void UpdateLastAppendStateForRemove(base::TimeDelta remove_start,
370 base::TimeDelta remove_end,
373 SourceBufferStreamType GetType() const;
375 // See GetNextBuffer() for additional details. This method handles preroll
377 SourceBufferStreamStatus HandleNextBufferWithPreroll(
378 scoped_refptr<StreamParserBuffer>* out_buffer);
380 // See GetNextBuffer() for additional details. The internal method hands out
381 // single buffers from the |track_buffer_| and |selected_range_| without
382 // additional processing for preroll buffers.
383 SourceBufferStreamStatus GetNextBufferInternal(
384 scoped_refptr<StreamParserBuffer>* out_buffer);
386 // If the next buffer's timestamp is significantly beyond the last output
387 // buffer, and if we just exhausted |track_buffer_| on the previous read, this
388 // method logs a warning to |media_log_| that there could be perceivable
389 // delay. Apps can avoid this behavior by not overlap-appending buffers near
390 // current playback position.
391 void WarnIfTrackBufferExhaustionSkipsForward(
392 scoped_refptr<StreamParserBuffer> next_buffer);
394 // If |out_buffer| has preroll, sets |pending_buffer_| to feed out preroll and
395 // returns true. Otherwise returns false.
396 bool SetPendingBuffer(scoped_refptr<StreamParserBuffer>* out_buffer);
398 // Used to report log messages that can help the web developer figure out what
399 // is wrong with the content.
400 raw_ptr<MediaLog> media_log_;
402 // List of disjoint buffered ranges, ordered by start time.
405 // Indicates which decoder config is being used by the decoder.
406 // GetNextBuffer() is only allows to return buffers that have a
407 // config ID that matches this index. If there is a mismatch then
408 // it must signal that a config change is needed.
409 int current_config_index_ = 0;
411 // Indicates which decoder config to associate with new buffers
412 // being appended. Each new buffer appended has its config ID set
413 // to the value of this field.
414 int append_config_index_ = 0;
416 // Holds the audio/video configs for this stream. |current_config_index_|
417 // and |append_config_index_| represent indexes into one of these vectors.
418 std::vector<AudioDecoderConfig> audio_configs_;
419 std::vector<VideoDecoderConfig> video_configs_;
421 // Holds the text config for this stream.
422 TextTrackConfig text_track_config_;
424 // True if more data needs to be appended before the Seek() can complete,
425 // false if no Seek() has been requested or the Seek() is completed.
426 bool seek_pending_ = false;
428 // True if the end of the stream has been signalled.
429 bool end_of_stream_ = false;
431 // Timestamp of the last request to Seek().
432 base::TimeDelta seek_buffer_timestamp_;
434 // Pointer to the seeked-to Range. This is the range from which
435 // GetNextBuffer() calls are fulfilled after the |track_buffer_| has been
437 raw_ptr<SourceBufferRange> selected_range_ = nullptr;
439 // Queue of the next buffers to be returned from calls to GetNextBuffer(). If
440 // |track_buffer_| is empty, return buffers from |selected_range_|.
441 BufferQueue track_buffer_;
443 // If there has been no intervening Seek, this will be true if the last
444 // emitted buffer emptied |track_buffer_|.
445 bool just_exhausted_track_buffer_ = false;
447 // The start presentation time of the current coded frame group being
449 base::TimeDelta coded_frame_group_start_pts_;
451 // Points to the range containing the current coded frame group being
453 RangeList::iterator range_for_next_append_;
455 // True when the next call to Append() begins a new coded frame group.
456 // TODO(wolenetz): Simplify by passing this flag into Append().
457 bool new_coded_frame_group_ = false;
459 // The timestamp of the last buffer appended to the coded frame group, set to
460 // kNoTimestamp if the beginning of the group.
461 base::TimeDelta last_appended_buffer_timestamp_ = kNoTimestamp;
462 base::TimeDelta last_appended_buffer_duration_ = kNoTimestamp;
463 bool last_appended_buffer_is_keyframe_ = false;
465 // When buffering GOPs by keyframe PTS and intra-gop by nonkeyframe DTS, to
466 // verify monotonically increasing intra-GOP DTS sequence and to update max
467 // interbuffer distance also by DTS deltas within a coded frame group, the
468 // following is needed.
469 DecodeTimestamp last_appended_buffer_decode_timestamp_ = kNoDecodeTimestamp;
471 // The following is the highest presentation timestamp appended so far in this
472 // coded frame group. Due to potentially out-of-order decode versus
473 // presentation time sequence, this isn't necessarily the most recently
474 // appended frame. This is used as the lower bound of removing previously
475 // buffered media when processing new appends.
476 base::TimeDelta highest_timestamp_in_append_sequence_ = kNoTimestamp;
478 // The following is used in determining if FreeBuffersAfterLastAppended() is
479 // allowed during garbage collection. Due to out-of-order decode versus
480 // presentation sequence, this isn't necessarily the end time of the most
481 // recently appended frame.
482 base::TimeDelta highest_buffered_end_time_in_append_sequence_ = kNoTimestamp;
484 // The highest presentation timestamp for buffers returned by recent
485 // GetNextBuffer() calls. Set to kNoTimestamp if GetNextBuffer() hasn't been
486 // called yet or a seek has happened since the last GetNextBuffer() call.
487 base::TimeDelta highest_output_buffer_timestamp_;
489 // Stores the largest distance between two adjacent buffers in this stream.
490 base::TimeDelta max_interbuffer_distance_;
492 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level_ =
493 base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE;
495 // The maximum amount of data in bytes the stream will keep in memory.
496 // |memory_limit_| is initialized based on the audio/video configuration in
497 // the constructor, but either user-setting of |memory_limit_| or
498 // memory-pressure-based adjustment to determine effective limit in the
499 // eviction heuristic can cause the result to vary from the value set in
501 size_t memory_limit_;
503 // Indicates that a kConfigChanged status has been reported by GetNextBuffer()
504 // and GetCurrentXXXDecoderConfig() must be called to update the current
505 // config. GetNextBuffer() must not be called again until
506 // GetCurrentXXXDecoderConfig() has been called.
507 bool config_change_pending_ = false;
509 // Used by HandleNextBufferWithPreroll() when a buffer with preroll is
510 // returned from GetNextBufferInternal().
511 scoped_refptr<StreamParserBuffer> pending_buffer_;
513 // Indicates that all buffers before |pending_buffer_| have been handed out.
514 bool pending_buffers_complete_ = false;
516 // To prevent log spam, count the number of logs for different log scenarios.
517 int num_splice_logs_ = 0;
518 int num_track_buffer_gap_warning_logs_ = 0;
519 int num_garbage_collect_algorithm_logs_ = 0;
524 #endif // MEDIA_FILTERS_SOURCE_BUFFER_STREAM_H_