1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_FILTERS_CHUNK_DEMUXER_H_
6 #define MEDIA_FILTERS_CHUNK_DEMUXER_H_
16 #include "base/containers/circular_deque.h"
17 #include "base/memory/memory_pressure_listener.h"
18 #include "base/memory/raw_ptr.h"
19 #include "base/synchronization/lock.h"
20 #include "base/thread_annotations.h"
21 #include "base/time/time.h"
22 #include "media/base/demuxer.h"
23 #include "media/base/demuxer_stream.h"
24 #include "media/base/media_tracks.h"
25 #include "media/base/ranges.h"
26 #include "media/base/stream_parser.h"
27 #include "media/filters/source_buffer_parse_warnings.h"
28 #include "media/filters/source_buffer_state.h"
29 #include "media/filters/source_buffer_stream.h"
31 class MEDIA_EXPORT SourceBufferStream;
35 class AudioDecoderConfig;
36 class VideoDecoderConfig;
38 class MEDIA_EXPORT ChunkDemuxerStream : public DemuxerStream {
40 using BufferQueue = base::circular_deque<scoped_refptr<StreamParserBuffer>>;
42 ChunkDemuxerStream() = delete;
44 ChunkDemuxerStream(Type type, MediaTrack::Id media_track_id);
46 ChunkDemuxerStream(const ChunkDemuxerStream&) = delete;
47 ChunkDemuxerStream& operator=(const ChunkDemuxerStream&) = delete;
49 ~ChunkDemuxerStream() override;
51 // ChunkDemuxerStream control methods.
52 void StartReturningData();
54 void CompletePendingReadIfPossible();
57 // SourceBufferStream manipulation methods.
58 void Seek(base::TimeDelta time);
59 bool IsSeekWaitingForData() const;
61 // Add buffers to this stream. Buffers are stored in SourceBufferStreams,
62 // which handle ordering and overlap resolution.
63 // Returns true if buffers were successfully added.
64 bool Append(const StreamParser::BufferQueue& buffers);
66 // Removes buffers between |start| and |end| according to the steps
67 // in the "Coded Frame Removal Algorithm" in the Media Source
69 // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#sourcebuffer-coded-frame-removal
71 // |duration| is the current duration of the presentation. It is
72 // required by the computation outlined in the spec.
73 void Remove(base::TimeDelta start, base::TimeDelta end,
74 base::TimeDelta duration);
76 // If the buffer is full, attempts to try to free up space, as specified in
77 // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec.
78 // Returns false iff buffer is still full after running eviction.
79 // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction
80 bool EvictCodedFrames(base::TimeDelta media_time, size_t newDataSize);
82 void OnMemoryPressure(
83 base::TimeDelta media_time,
84 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
85 bool force_instant_gc);
87 // Signal to the stream that duration has changed to |duration|.
88 void OnSetDuration(base::TimeDelta duration);
90 // Returns the range of buffered data in this stream, capped at |duration|.
91 Ranges<base::TimeDelta> GetBufferedRanges(base::TimeDelta duration) const;
93 // Returns the highest PTS of the buffered data.
94 // Returns base::TimeDelta() if the stream has no buffered data.
95 base::TimeDelta GetHighestPresentationTimestamp() const;
97 // Returns the duration of the buffered data.
98 // Returns base::TimeDelta() if the stream has no buffered data.
99 base::TimeDelta GetBufferedDuration() const;
101 // Returns the size of the buffered data in bytes.
102 size_t GetBufferedSize() const;
104 // Signal to the stream that buffers handed in through subsequent calls to
105 // Append() belong to a coded frame group that starts at |start_pts|.
106 // |start_dts| is used only to help tests verify correctness of calls to this
107 // method. If |group_start_observer_cb_| is set, first invokes this test-only
108 // callback with |start_dts| and |start_pts| to assist test verification.
109 void OnStartOfCodedFrameGroup(DecodeTimestamp start_dts,
110 base::TimeDelta start_pts);
112 // Called when midstream config updates occur.
113 // For audio and video, if the codec is allowed to change, the caller should
114 // set |allow_codec_change| to true.
115 // Returns true if the new config is accepted.
116 // Returns false if the new config should trigger an error.
117 bool UpdateAudioConfig(const AudioDecoderConfig& config,
118 bool allow_codec_change,
119 MediaLog* media_log);
120 bool UpdateVideoConfig(const VideoDecoderConfig& config,
121 bool allow_codec_change,
122 MediaLog* media_log);
123 void UpdateTextConfig(const TextTrackConfig& config, MediaLog* media_log);
125 void MarkEndOfStream();
126 void UnmarkEndOfStream();
128 // DemuxerStream methods.
129 void Read(ReadCB read_cb) override;
130 Type type() const override;
131 StreamLiveness liveness() const override;
132 AudioDecoderConfig audio_decoder_config() override;
133 VideoDecoderConfig video_decoder_config() override;
134 bool SupportsConfigChanges() override;
136 bool IsEnabled() const;
137 void SetEnabled(bool enabled, base::TimeDelta timestamp);
139 // Returns the text track configuration. It is an error to call this method
140 // if type() != TEXT.
141 TextTrackConfig text_track_config();
143 // Sets the memory limit, in bytes, on the SourceBufferStream.
144 void SetStreamMemoryLimit(size_t memory_limit);
146 void SetLiveness(StreamLiveness liveness);
148 MediaTrack::Id media_track_id() const { return media_track_id_; }
150 // Allows tests to verify invocations of Append().
151 using AppendObserverCB = base::RepeatingCallback<void(const BufferQueue*)>;
152 void set_append_observer_for_testing(AppendObserverCB append_observer_cb) {
153 append_observer_cb_ = std::move(append_observer_cb);
156 // Allows tests to verify invocations of OnStartOfCodedFrameGroup().
157 using GroupStartObserverCB =
158 base::RepeatingCallback<void(DecodeTimestamp, base::TimeDelta)>;
159 void set_group_start_observer_for_testing(
160 GroupStartObserverCB group_start_observer_cb) {
161 group_start_observer_cb_ = std::move(group_start_observer_cb);
167 RETURNING_DATA_FOR_READS,
168 RETURNING_ABORT_FOR_READS,
172 // Assigns |state_| to |state|
173 void ChangeState_Locked(State state) EXCLUSIVE_LOCKS_REQUIRED(lock_);
175 void CompletePendingReadIfPossible_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_);
177 // Specifies the type of the stream.
180 StreamLiveness liveness_ GUARDED_BY(lock_);
182 std::unique_ptr<SourceBufferStream> stream_ GUARDED_BY(lock_);
184 const MediaTrack::Id media_track_id_;
186 // Test-only callbacks to assist verification of Append() and
187 // OnStartOfCodedFrameGroup() calls, respectively.
188 AppendObserverCB append_observer_cb_;
189 GroupStartObserverCB group_start_observer_cb_;
191 mutable base::Lock lock_;
192 State state_ GUARDED_BY(lock_);
193 ReadCB read_cb_ GUARDED_BY(lock_);
194 bool is_enabled_ GUARDED_BY(lock_);
197 // Demuxer implementation that allows chunks of media data to be passed
198 // from JavaScript to the media stack.
199 class MEDIA_EXPORT ChunkDemuxer : public Demuxer {
202 kOk, // ID added w/o error.
203 kNotSupported, // Type specified is not supported.
204 kReachedIdLimit, // Reached ID limit. We can't handle any more IDs.
207 // |open_cb| Run when Initialize() is called to signal that the demuxer
208 // is ready to receive media data via AppendData/Chunks().
209 // |progress_cb| Run each time data is appended.
210 // |encrypted_media_init_data_cb| Run when the demuxer determines that an
211 // encryption key is needed to decrypt the content.
212 // |media_log| Used to report content and engine debug messages.
213 ChunkDemuxer(base::OnceClosure open_cb,
214 base::RepeatingClosure progress_cb,
215 EncryptedMediaInitDataCB encrypted_media_init_data_cb,
216 MediaLog* media_log);
218 ChunkDemuxer(const ChunkDemuxer&) = delete;
219 ChunkDemuxer& operator=(const ChunkDemuxer&) = delete;
221 ~ChunkDemuxer() override;
223 // Demuxer implementation.
224 std::string GetDisplayName() const override;
226 // |enable_text| Process inband text tracks in the normal way when true,
227 // otherwise ignore them.
228 void Initialize(DemuxerHost* host, PipelineStatusCallback init_cb) override;
229 void Stop() override;
230 void Seek(base::TimeDelta time, PipelineStatusCallback cb) override;
231 base::Time GetTimelineOffset() const override;
232 std::vector<DemuxerStream*> GetAllStreams() override;
233 base::TimeDelta GetStartTime() const override;
234 int64_t GetMemoryUsage() const override;
235 absl::optional<container_names::MediaContainerName> GetContainerForMetrics()
237 void AbortPendingReads() override;
239 // ChunkDemuxer reads are abortable. StartWaitingForSeek() and
240 // CancelPendingSeek() always abort pending and future reads until the
241 // expected seek occurs, so that ChunkDemuxer can stay synchronized with the
242 // associated JS method calls.
243 void StartWaitingForSeek(base::TimeDelta seek_time) override;
244 void CancelPendingSeek(base::TimeDelta seek_time) override;
246 // Registers a new |id| to use for AppendData/Chunks() calls. |content_type|
247 // indicates the MIME type's ContentType and |codecs| indicates the MIME
248 // type's "codecs" parameter string (if any) for the data that we intend to
249 // append for this ID. kOk is returned if the demuxer has enough resources to
250 // support another ID and supports the format indicated by |content_type| and
251 // |codecs|. kReachedIdLimit is returned if the demuxer cannot handle another
252 // ID right now. kNotSupported is returned if |content_type| and |codecs| is
253 // not a supported format.
254 // The |audio_config| and |video_config| overloads behave similarly, except
255 // the caller must provide valid, supported decoder configs; those overloads'
256 // usage indicates that we intend to append WebCodecs encoded audio or video
257 // chunks for this ID.
258 [[nodiscard]] Status AddId(const std::string& id,
259 const std::string& content_type,
260 const std::string& codecs);
261 [[nodiscard]] Status AddId(const std::string& id,
262 std::unique_ptr<AudioDecoderConfig> audio_config);
263 [[nodiscard]] Status AddId(const std::string& id,
264 std::unique_ptr<VideoDecoderConfig> video_config);
266 // Notifies a caller via |tracks_updated_cb| that the set of media tracks
267 // for a given |id| has changed. This callback must be set before any calls to
268 // AppendData() for this |id|.
269 void SetTracksWatcher(const std::string& id,
270 MediaTracksUpdatedCB tracks_updated_cb);
272 // Notifies a caller via |parse_warning_cb| of a parse warning. This callback
273 // must be set before any calls to AppendData() for this |id|.
274 void SetParseWarningCallback(const std::string& id,
275 SourceBufferParseWarningCB parse_warning_cb);
277 // Removed an ID & associated resources that were previously added with
279 void RemoveId(const std::string& id);
281 // Gets the currently buffered ranges for the specified ID.
282 Ranges<base::TimeDelta> GetBufferedRanges(const std::string& id) const;
284 // Gets the highest buffered PTS for the specified |id|. If there is nothing
285 // buffered, returns base::TimeDelta().
286 base::TimeDelta GetHighestPresentationTimestamp(const std::string& id) const;
288 void OnEnabledAudioTracksChanged(const std::vector<MediaTrack::Id>& track_ids,
289 base::TimeDelta curr_time,
290 TrackChangeCB change_completed_cb) override;
292 void OnSelectedVideoTrackChanged(const std::vector<MediaTrack::Id>& track_ids,
293 base::TimeDelta curr_time,
294 TrackChangeCB change_completed_cb) override;
296 // Appends media data to the source buffer associated with |id|, applying
297 // and possibly updating |*timestamp_offset| during coded frame processing.
298 // |append_window_start| and |append_window_end| correspond to the MSE spec's
299 // similarly named source buffer attributes that are used in coded frame
300 // processing. Returns true on success, false if the caller needs to run the
301 // append error algorithm with decode error parameter set to true.
302 [[nodiscard]] bool AppendData(const std::string& id,
305 base::TimeDelta append_window_start,
306 base::TimeDelta append_window_end,
307 base::TimeDelta* timestamp_offset);
309 // Appends webcodecs encoded chunks (already converted by caller into a
310 // BufferQueue of StreamParserBuffers) to the source buffer associated with
311 // |id|, with same semantic for other parameters and return value as
313 [[nodiscard]] bool AppendChunks(
314 const std::string& id,
315 std::unique_ptr<StreamParser::BufferQueue> buffer_queue,
316 base::TimeDelta append_window_start,
317 base::TimeDelta append_window_end,
318 base::TimeDelta* timestamp_offset);
320 // Aborts parsing the current segment and reset the parser to a state where
321 // it can accept a new segment.
322 // Some pending frames can be emitted during that process. These frames are
323 // applied |timestamp_offset|.
324 void ResetParserState(const std::string& id,
325 base::TimeDelta append_window_start,
326 base::TimeDelta append_window_end,
327 base::TimeDelta* timestamp_offset);
329 // Remove buffers between |start| and |end| for the source buffer
330 // associated with |id|.
331 void Remove(const std::string& id, base::TimeDelta start,
332 base::TimeDelta end);
334 // Returns whether or not the source buffer associated with |id| can change
335 // its parser type to one which parses |content_type| and |codecs|.
336 // |content_type| indicates the ContentType of the MIME type for the data that
337 // we intend to append for this |id|; |codecs| similarly indicates the MIME
338 // type's "codecs" parameter, if any.
339 bool CanChangeType(const std::string& id,
340 const std::string& content_type,
341 const std::string& codecs);
343 // For the source buffer associated with |id|, changes its parser type to one
344 // which parses |content_type| and |codecs|. |content_type| indicates the
345 // ContentType of the MIME type for the data that we intend to append for this
346 // |id|; |codecs| similarly indicates the MIME type's "codecs" parameter, if
347 // any. Caller must first ensure CanChangeType() returns true for the same
348 // parameters. Caller must also ensure that ResetParserState() is done before
349 // calling this, to flush any pending frames.
350 void ChangeType(const std::string& id,
351 const std::string& content_type,
352 const std::string& codecs);
354 // If the buffer is full, attempts to try to free up space, as specified in
355 // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec.
356 // Returns false iff buffer is still full after running eviction.
357 // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction
358 [[nodiscard]] bool EvictCodedFrames(const std::string& id,
359 base::TimeDelta currentMediaTime,
362 void OnMemoryPressure(
363 base::TimeDelta currentMediaTime,
364 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
365 bool force_instant_gc);
367 // Returns the current presentation duration.
368 double GetDuration();
369 double GetDuration_Locked();
371 // Notifies the demuxer that the duration of the media has changed to
373 void SetDuration(double duration);
375 // Returns true if the source buffer associated with |id| is currently parsing
376 // a media segment, or false otherwise.
377 bool IsParsingMediaSegment(const std::string& id);
379 // Returns the 'Generate Timestamps Flag', as described in the MSE Byte Stream
380 // Format Registry, for the source buffer associated with |id|.
381 bool GetGenerateTimestampsFlag(const std::string& id);
383 // Set the append mode to be applied to subsequent buffers appended to the
384 // source buffer associated with |id|. If |sequence_mode| is true, caller
385 // is requesting "sequence" mode. Otherwise, caller is requesting "segments"
387 void SetSequenceMode(const std::string& id, bool sequence_mode);
389 // Signals the coded frame processor for the source buffer associated with
390 // |id| to update its group start timestamp to be |timestamp_offset| if it is
391 // in sequence append mode.
392 void SetGroupStartTimestampIfInSequenceMode(const std::string& id,
393 base::TimeDelta timestamp_offset);
395 // Called to signal changes in the "end of stream"
396 // state. UnmarkEndOfStream() must not be called if a matching
397 // MarkEndOfStream() has not come before it.
398 void MarkEndOfStream(PipelineStatus status);
399 void UnmarkEndOfStream();
403 // Sets the memory limit on each stream of a specific type.
404 // |memory_limit| is the maximum number of bytes each stream of type |type|
405 // is allowed to hold in its buffer.
406 void SetMemoryLimitsForTest(DemuxerStream::Type type, size_t memory_limit);
408 // Returns the ranges representing the buffered data in the demuxer.
409 // TODO(wolenetz): Remove this method once MediaSourceDelegate no longer
410 // requires it for doing hack browser seeks to I-frame on Android. See
411 // http://crbug.com/304234.
412 Ranges<base::TimeDelta> GetBufferedRanges() const;
416 WAITING_FOR_INIT = 0,
421 // Any State at or beyond PARSE_ERROR cannot be changed to a state before
422 // this. See ChangeState_Locked.
427 // Helper for AddId's creation of FrameProcessor, and
428 // SourceBufferState creation, initialization and tracking in
429 // source_state_map_.
430 ChunkDemuxer::Status AddIdInternal(
431 const std::string& id,
432 std::unique_ptr<media::StreamParser> stream_parser,
433 std::string expected_codecs);
435 // Helper for vide and audio track changing.
436 void FindAndEnableProperTracks(const std::vector<MediaTrack::Id>& track_ids,
437 base::TimeDelta curr_time,
438 DemuxerStream::Type track_type,
439 TrackChangeCB change_completed_cb);
441 void ChangeState_Locked(State new_state);
443 // Reports an error and puts the demuxer in a state where it won't accept more
445 void ReportError_Locked(PipelineStatus error);
447 // Returns true if any stream has seeked to a time without buffered data.
448 bool IsSeekWaitingForData_Locked() const;
450 // Returns true if all streams can successfully call EndOfStream,
451 // false if any can not.
452 bool CanEndOfStream_Locked() const;
454 // SourceBufferState callbacks.
455 void OnSourceInitDone(const std::string& source_id,
456 const StreamParser::InitParameters& params);
458 // Creates a DemuxerStream of the specified |type| for the SourceBufferState
459 // with the given |source_id|.
460 // Returns a pointer to a new ChunkDemuxerStream instance, which is owned by
462 ChunkDemuxerStream* CreateDemuxerStream(const std::string& source_id,
463 DemuxerStream::Type type);
465 void OnNewTextTrack(ChunkDemuxerStream* text_stream,
466 const TextTrackConfig& config);
468 // Returns true if |source_id| is valid, false otherwise.
469 bool IsValidId(const std::string& source_id) const;
471 // Increases |duration_| to |new_duration|, if |new_duration| is higher.
472 void IncreaseDurationIfNecessary(base::TimeDelta new_duration);
474 // Decreases |duration_| if the buffered region is less than |duration_| when
475 // EndOfStream() is called.
476 void DecreaseDurationIfNecessary();
478 // Sets |duration_| to |new_duration|, sets |user_specified_duration_| to -1
479 // and notifies |host_|.
480 void UpdateDuration(base::TimeDelta new_duration);
482 // Returns the ranges representing the buffered data in the demuxer.
483 Ranges<base::TimeDelta> GetBufferedRanges_Locked() const;
485 // Start returning data on all DemuxerStreams.
486 void StartReturningData();
488 void AbortPendingReads_Locked();
490 // Completes any pending reads if it is possible to do so.
491 void CompletePendingReadsIfPossible();
493 // Seeks all SourceBufferStreams to |seek_time|.
494 void SeekAllSources(base::TimeDelta seek_time);
496 // Generates and returns a unique media track id.
497 static MediaTrack::Id GenerateMediaTrackId();
499 // Shuts down all DemuxerStreams by calling Shutdown() on
500 // all objects in |source_state_map_|.
501 void ShutdownAllStreams();
503 // Executes |init_cb_| with |status| and closes out the async trace.
504 void RunInitCB_Locked(PipelineStatus status);
506 // Executes |seek_cb_| with |status| and closes out the async trace.
507 void RunSeekCB_Locked(PipelineStatus status);
509 mutable base::Lock lock_;
511 bool cancel_next_seek_;
513 raw_ptr<DemuxerHost> host_;
514 base::OnceClosure open_cb_;
515 const base::RepeatingClosure progress_cb_;
516 EncryptedMediaInitDataCB encrypted_media_init_data_cb_;
518 // MediaLog for reporting messages and properties to debug content and engine.
519 raw_ptr<MediaLog> media_log_;
521 PipelineStatusCallback init_cb_;
522 // Callback to execute upon seek completion.
523 // TODO(wolenetz/acolwell): Protect against possible double-locking by first
524 // releasing |lock_| before executing this callback. See
525 // http://crbug.com/308226
526 PipelineStatusCallback seek_cb_;
528 using OwnedChunkDemuxerStreamVector =
529 std::vector<std::unique_ptr<ChunkDemuxerStream>>;
530 OwnedChunkDemuxerStreamVector audio_streams_;
531 OwnedChunkDemuxerStreamVector video_streams_;
532 OwnedChunkDemuxerStreamVector text_streams_;
534 // Keep track of which ids still remain uninitialized so that we transition
535 // into the INITIALIZED only after all ids/SourceBuffers got init segment.
536 std::set<std::string> pending_source_init_ids_;
538 base::TimeDelta duration_;
540 // The duration passed to the last SetDuration(). If
541 // SetDuration() is never called or an AppendData/Chunks() call or
542 // a EndOfStream() call changes |duration_|, then this
543 // variable is set to < 0 to indicate that the |duration_| represents
544 // the actual duration instead of a user specified value.
545 double user_specified_duration_;
547 base::Time timeline_offset_;
548 StreamLiveness liveness_ = StreamLiveness::kUnknown;
550 std::map<std::string, std::unique_ptr<SourceBufferState>> source_state_map_;
552 std::map<std::string, std::vector<ChunkDemuxerStream*>> id_to_streams_map_;
553 // Used to hold alive the demuxer streams that were created for removed /
554 // released SourceBufferState objects. Demuxer clients might still have
555 // references to these streams, so we need to keep them alive. But they'll be
556 // in a shut down state, so reading from them will return EOS.
557 std::vector<std::unique_ptr<ChunkDemuxerStream>> removed_streams_;
559 std::map<MediaTrack::Id, ChunkDemuxerStream*> track_id_to_demux_stream_map_;
564 #endif // MEDIA_FILTERS_CHUNK_DEMUXER_H_