1 // Copyright 2016 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_
6 #define MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_
8 #include "base/functional/bind.h"
9 #include "base/memory/memory_pressure_listener.h"
10 #include "base/memory/raw_ptr.h"
11 #include "base/time/time.h"
12 #include "media/base/audio_codecs.h"
13 #include "media/base/demuxer.h"
14 #include "media/base/demuxer_stream.h"
15 #include "media/base/media_export.h"
16 #include "media/base/media_log.h"
17 #include "media/base/stream_parser.h"
18 #include "media/base/stream_parser_buffer.h"
19 #include "media/base/video_codecs.h"
20 #include "media/filters/source_buffer_parse_warnings.h"
25 class ChunkDemuxerStream;
28 // Contains state belonging to a source id.
29 class MEDIA_EXPORT SourceBufferState {
31 // Callback signature used to create ChunkDemuxerStreams.
32 using CreateDemuxerStreamCB =
33 base::RepeatingCallback<ChunkDemuxerStream*(DemuxerStream::Type)>;
35 SourceBufferState(std::unique_ptr<StreamParser> stream_parser,
36 std::unique_ptr<FrameProcessor> frame_processor,
37 CreateDemuxerStreamCB create_demuxer_stream_cb,
40 SourceBufferState(const SourceBufferState&) = delete;
41 SourceBufferState& operator=(const SourceBufferState&) = delete;
45 void Init(StreamParser::InitCB init_cb,
46 const std::string& expected_codecs,
47 #if BUILDFLAG(IS_TIZEN_TV)
48 const StreamParser::FramerateSetCB& framerate_set_cb,
50 const StreamParser::EncryptedMediaInitDataCB&
51 encrypted_media_init_data_cb);
53 // Reconfigures this source buffer to use |new_stream_parser|. Caller must
54 // first ensure that ResetParserState() was done to flush any pending frames
55 // from the old stream parser.
56 void ChangeType(std::unique_ptr<StreamParser> new_stream_parser,
57 #if BUILDFLAG(IS_TIZEN_TV)
58 const StreamParser::FramerateSetCB& framerate_set_cb,
60 const std::string& new_expected_codecs);
62 // Appends media data to the StreamParser, but no parsing is done of it yet,
63 // just buffering the media data for future parsing via RunSegmentParserLoop()
64 // calls. Returns true on success. Returns false if the parser was unable to
65 // allocate resources; content in `data` is not copied as a result, and this
66 // failure is reported (through various layers) up to the SourceBuffer's
67 // implementation of appendBuffer(), which should then notify the app of
68 // append failure using a `QuotaExceededErr` exception per the MSE
69 // specification. App could use a back-off and retry strategy or otherwise
70 // alter their behavior to attempt to buffer media for further playback.
71 [[nodiscard]] bool AppendToParseBuffer(const uint8_t* data, size_t length);
73 // Tells the stream parser to parse more of the data previously sent to it
74 // from this object's AppendToParseBuffer(). `*timestamp_offset` is used and
75 // possibly updated by the parsing. `append_window_start` and
76 // `append_window_end` correspond to the MSE spec's similarly named source
77 // buffer attributes that are used in coded frame processing.
78 // Returns kSuccess if the parse succeeded and all previously provided data
79 // from AppendToParseBuffer() has been inspected.
80 // Returns kSuccessHasMoreData if the parse succeeded, yet there remains
81 // uninspected data remaining from AppendToParseBuffer(); more call(s) to this
82 // method are necessary for the parser to attempt inspection of that data.
83 // Returns kFailed if the parse failed.
84 [[nodiscard]] StreamParser::ParseStatus RunSegmentParserLoop(
85 base::TimeDelta append_window_start,
86 base::TimeDelta append_window_end,
87 base::TimeDelta* timestamp_offset);
89 // AppendChunks appends the provided BufferQueue.
90 [[nodiscard]] bool AppendChunks(
91 std::unique_ptr<StreamParser::BufferQueue> buffer_queue,
92 base::TimeDelta append_window_start,
93 base::TimeDelta append_window_end,
94 base::TimeDelta* timestamp_offset);
96 // Aborts the current append sequence and resets the parser.
97 void ResetParserState(base::TimeDelta append_window_start,
98 base::TimeDelta append_window_end,
99 base::TimeDelta* timestamp_offset);
101 // Calls Remove(|start|, |end|, |duration|) on all
102 // ChunkDemuxerStreams managed by this object.
103 void Remove(base::TimeDelta start,
105 base::TimeDelta duration);
107 // If the buffer is full, attempts to try to free up space, as specified in
108 // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec.
109 // Returns false iff buffer is still full after running eviction.
110 // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction
111 bool EvictCodedFrames(base::TimeDelta media_time, size_t newDataSize);
113 // Gets invoked when the system is experiencing memory pressure, i.e. there's
114 // not enough free memory. The |media_time| is the media playback position at
115 // the time of memory pressure notification (needed for accurate GC). The
116 // |memory_pressure_level| indicates memory pressure severity. The
117 // |force_instant_gc| is used to force the MSE garbage collection algorithm to
118 // be run right away, without waiting for the next append.
119 void OnMemoryPressure(
120 base::TimeDelta media_time,
121 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
122 bool force_instant_gc);
124 // Returns true if currently parsing a media segment, or false otherwise.
125 bool parsing_media_segment() const { return parsing_media_segment_; }
127 // Returns the 'Generate Timestamps Flag' for this SourceBuffer's byte stream
128 // format parser as described in the MSE Byte Stream Format Registry.
129 bool generate_timestamps_flag() const {
130 return stream_parser_->GetGenerateTimestampsFlag();
133 // Sets |frame_processor_|'s sequence mode to |sequence_mode|.
134 void SetSequenceMode(bool sequence_mode);
136 // Signals the coded frame processor to update its group start timestamp to be
137 // |timestamp_offset| if it is in sequence append mode.
138 void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset);
140 // Returns the range of buffered data in this source, capped at |duration|.
141 // |ended| - Set to true if end of stream has been signaled and the special
142 // end of stream range logic needs to be executed.
143 Ranges<base::TimeDelta> GetBufferedRanges(base::TimeDelta duration,
146 // Returns the lowest PTS of currently buffered frames in this source, or
147 // base::TimeDelta() if none of the streams contain buffered data.
148 base::TimeDelta GetLowestPresentationTimestamp() const;
150 // Returns the highest PTS of currently buffered frames in this source, or
151 // base::TimeDelta() if none of the streams contain buffered data.
152 base::TimeDelta GetHighestPresentationTimestamp() const;
154 // Returns the highest buffered duration across all streams managed
156 // Returns base::TimeDelta() if none of the streams contain buffered data.
157 base::TimeDelta GetMaxBufferedDuration() const;
159 // Helper methods that call methods with similar names on all the
160 // ChunkDemuxerStreams managed by this object.
161 void StartReturningData();
163 void Seek(base::TimeDelta seek_time);
164 void CompletePendingReadIfPossible();
165 void OnSetDuration(base::TimeDelta duration);
166 void MarkEndOfStream();
167 void UnmarkEndOfStream();
169 // Sets the memory limit on each stream of a specific type.
170 // |memory_limit| is the maximum number of bytes each stream of type |type|
171 // is allowed to hold in its buffer.
172 void SetMemoryLimits(DemuxerStream::Type type, size_t memory_limit);
173 bool IsSeekWaitingForData() const;
175 using RangesList = std::vector<Ranges<base::TimeDelta>>;
176 static Ranges<base::TimeDelta> ComputeRangesIntersection(
177 const RangesList& active_ranges,
180 void SetTracksWatcher(Demuxer::MediaTracksUpdatedCB tracks_updated_cb);
182 void SetParseWarningCallback(SourceBufferParseWarningCB parse_warning_cb);
185 // State advances through this list to PARSER_INITIALIZED.
186 // The intent is to ensure at least one config is received prior to parser
187 // calling initialization callback, and that such initialization callback
188 // occurs at most once per parser.
189 // PENDING_PARSER_RECONFIG occurs if State had reached PARSER_INITIALIZED
190 // before changing to a new StreamParser in ChangeType(). In such case, State
191 // would then advance to PENDING_PARSER_REINIT, then PARSER_INITIALIZED upon
192 // the next initialization segment parsed, but would not run the
193 // initialization callback in this case (since such would already have
194 // occurred on the initial transition from PENDING_PARSER_INIT to
195 // PARSER_INITIALIZED.)
198 PENDING_PARSER_CONFIG,
201 PENDING_PARSER_RECONFIG,
202 PENDING_PARSER_REINIT
205 // Initializes |stream_parser_|. Also, updates |expected_audio_codecs| and
206 // |expected_video_codecs|.
207 void InitializeParser(const std::string& expected_codecs);
209 // Called by the |stream_parser_| when a new initialization segment is
211 // Returns true on a successful call. Returns false if an error occurred while
212 // processing decoder configurations.
213 bool OnNewConfigs(std::string expected_codecs,
214 std::unique_ptr<MediaTracks> tracks);
216 // Called by the |stream_parser_| at the beginning of a new media segment.
217 void OnNewMediaSegment();
219 // Called by the |stream_parser_| at the end of a media segment.
220 void OnEndOfMediaSegment();
222 // Called by the |stream_parser_| when new buffers have been parsed.
223 // It processes the new buffers using |frame_processor_|, which includes
224 // appending the processed frames to associated demuxer streams for each
226 // Returns true on a successful call. Returns false if an error occurred while
227 // processing the buffers.
228 bool OnNewBuffers(const StreamParser::BufferQueueMap& buffer_queue_map);
230 // Called when StreamParser encounters encrypted media init data.
231 void OnEncryptedMediaInitData(EmeInitDataType type,
232 const std::vector<uint8_t>& init_data);
234 void OnSourceInitDone(const StreamParser::InitParameters& params);
236 // Sets memory limits for all demuxer streams.
237 void SetStreamMemoryLimits();
239 // Tracks the number of MEDIA_LOGs emitted for segments missing expected audio
240 // or video blocks. Useful to prevent log spam.
241 int num_missing_track_logs_ = 0;
243 // During RunSegmentParserLoop() or AppendChunks(), if OnNewBuffers() coded
244 // frame processing updates the timestamp offset then
245 // `*timestamp_offset_during_append_` is also updated so the caller can know
246 // the new offset. This pointer is only non-NULL during the lifetime of a
247 // RunSegmentParserLoop() or AppendChunks() call.
248 raw_ptr<base::TimeDelta> timestamp_offset_during_append_;
250 // During RunSegmentParserLoop() or AppendChunks(), coded frame processing
251 // triggered by OnNewBuffers() requires these two attributes. These are only
252 // valid during the lifetime of a RunSegmentParserLoop() or AppendChunks()
254 base::TimeDelta append_window_start_during_append_;
255 base::TimeDelta append_window_end_during_append_;
257 // Keeps track of whether a media segment is being parsed.
258 bool parsing_media_segment_;
260 // Valid only while |parsing_media_segment_| is true. These flags enable
261 // warning when the parsed media segment doesn't have frames for some track.
262 std::map<StreamParser::TrackId, bool> media_segment_has_data_for_track_;
264 // The object used to parse appended data.
265 std::unique_ptr<StreamParser> stream_parser_;
267 // Note that ChunkDemuxerStreams are created and owned by the parent
268 // ChunkDemuxer. They are not owned by |this|.
269 using DemuxerStreamMap = std::map<StreamParser::TrackId, ChunkDemuxerStream*>;
270 DemuxerStreamMap audio_streams_;
271 DemuxerStreamMap video_streams_;
273 std::unique_ptr<FrameProcessor> frame_processor_;
274 const CreateDemuxerStreamCB create_demuxer_stream_cb_;
275 raw_ptr<MediaLog> media_log_;
277 StreamParser::InitCB init_cb_;
278 StreamParser::EncryptedMediaInitDataCB encrypted_media_init_data_cb_;
282 // During RunSegmentParserLoop() or AppendChunks(), OnNewConfigs() will
283 // trigger the initialization segment received algorithm. Note, the MSE spec
284 // explicitly disallows this algorithm during an Abort(), since Abort() is
285 // allowed only to emit coded frames, and only if the parser is
286 // PARSING_MEDIA_SEGMENT (not an INIT segment). So we also have a
287 // `new_configs_possible_` flag here that indicates if RunSegmentParserLoop()
288 // or AppendChunks() is in progress and we can invoke this callback.
289 Demuxer::MediaTracksUpdatedCB init_segment_received_cb_;
290 bool new_configs_possible_ = false;
291 bool first_init_segment_received_ = false;
292 bool encrypted_media_init_data_reported_ = false;
294 std::vector<AudioCodec> expected_audio_codecs_;
295 std::vector<VideoCodec> expected_video_codecs_;
300 #endif // MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_