1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // Implements the Demuxer interface using FFmpeg's libavformat. At this time
6 // will support demuxing any audio/video format thrown at it. The streams
7 // output mime types audio/x-ffmpeg and video/x-ffmpeg and include an integer
8 // key FFmpegCodecID which contains the CodecID enumeration value. The CodecIDs
9 // can be used to create and initialize the corresponding FFmpeg decoder.
11 // FFmpegDemuxer sets the duration of pipeline during initialization by using
12 // the duration of the longest audio/video stream.
14 // NOTE: since FFmpegDemuxer reads packets sequentially without seeking, media
15 // files with very large drift between audio/video streams may result in
16 // excessive memory consumption.
18 // When stopped, FFmpegDemuxer and FFmpegDemuxerStream release all callbacks
19 // and buffered packets. Reads from a stopped FFmpegDemuxerStream will not be
22 #ifndef MEDIA_FILTERS_FFMPEG_DEMUXER_H_
23 #define MEDIA_FILTERS_FFMPEG_DEMUXER_H_
33 #include "base/functional/callback.h"
34 #include "base/memory/raw_ptr.h"
35 #include "base/memory/weak_ptr.h"
36 #include "base/task/sequenced_task_runner.h"
37 #include "base/time/time.h"
38 #include "media/base/audio_decoder_config.h"
39 #include "media/base/decoder_buffer.h"
40 #include "media/base/decoder_buffer_queue.h"
41 #include "media/base/demuxer.h"
42 #include "media/base/media_log.h"
43 #include "media/base/pipeline_status.h"
44 #include "media/base/timestamp_constants.h"
45 #include "media/base/video_decoder_config.h"
46 #include "media/ffmpeg/scoped_av_packet.h"
47 #include "media/filters/blocking_url_protocol.h"
48 #include "media/media_buildflags.h"
50 // FFmpeg forward declarations.
51 struct AVFormatContext;
58 class FFmpegBitstreamConverter;
62 class MEDIA_EXPORT FFmpegDemuxerStream : public DemuxerStream {
64 // Attempts to create FFmpegDemuxerStream form the given AVStream. Will return
65 // null if the AVStream cannot be translated into a valid decoder config.
67 // FFmpegDemuxerStream keeps a copy of |demuxer| and initializes itself using
68 // information inside |stream|. Both parameters must outlive |this|.
69 static std::unique_ptr<FFmpegDemuxerStream> Create(FFmpegDemuxer* demuxer,
73 FFmpegDemuxerStream(const FFmpegDemuxerStream&) = delete;
74 FFmpegDemuxerStream& operator=(const FFmpegDemuxerStream&) = delete;
76 ~FFmpegDemuxerStream() override;
78 // Enqueues the given AVPacket. It is invalid to queue a |packet| after
79 // SetEndOfStream() has been called.
80 void EnqueuePacket(ScopedAVPacket packet);
82 // Enters the end of stream state. After delivering remaining queued buffers
83 // only end of stream buffers will be delivered.
84 void SetEndOfStream();
86 // Drops queued buffers and clears end of stream state.
87 // Passing |preserve_packet_position| will prevent replay of already seen
89 void FlushBuffers(bool preserve_packet_position);
91 // Empties the queues and ignores any additional calls to Read().
94 // Aborts any pending reads.
97 base::TimeDelta duration() const { return duration_; }
99 // Enables fixes for files with negative timestamps. Normally all timestamps
100 // are rebased against FFmpegDemuxer::start_time() whenever that value is
101 // negative. When this fix is enabled, only AUDIO stream packets will be
102 // rebased to time zero, all other stream types will use the muxed timestamp.
104 // Further, when no codec delay is present, all AUDIO packets which originally
105 // had negative timestamps will be marked for post-decode discard. When codec
106 // delay is present, it is assumed the decoder will handle discard and does
107 // not need the AUDIO packets to be marked for discard; just rebased to zero.
108 void enable_negative_timestamp_fixups() {
109 fixup_negative_timestamps_ = true;
111 void enable_chained_ogg_fixups() { fixup_chained_ogg_ = true; }
113 // DemuxerStream implementation.
114 Type type() const override;
115 StreamLiveness liveness() const override;
116 void Read(uint32_t count, ReadCB read_cb) override;
117 void EnableBitstreamConverter() override;
118 bool SupportsConfigChanges() override;
119 AudioDecoderConfig audio_decoder_config() override;
120 VideoDecoderConfig video_decoder_config() override;
122 bool IsEnabled() const;
123 void SetEnabled(bool enabled, base::TimeDelta timestamp);
125 void SetLiveness(StreamLiveness liveness);
127 // Returns the range of buffered data in this stream.
128 Ranges<base::TimeDelta> GetBufferedRanges() const;
130 // Returns true if this stream has capacity for additional data.
131 bool HasAvailableCapacity();
133 // Returns the total buffer size FFMpegDemuxerStream is holding onto.
134 size_t MemoryUsage() const;
136 // Returns the value associated with |key| in the metadata for the avstream.
137 // Returns an empty string if the key is not present.
138 std::string GetMetadata(const char* key) const;
140 AVStream* av_stream() const { return stream_; }
142 base::TimeDelta start_time() const { return start_time_; }
143 void set_start_time(base::TimeDelta time) { start_time_ = time; }
146 friend class FFmpegDemuxerTest;
148 // Use FFmpegDemuxerStream::Create to construct.
149 // Audio/Video streams must include their respective DecoderConfig. At most
150 // one DecoderConfig should be provided (leaving the other nullptr).
151 FFmpegDemuxerStream(FFmpegDemuxer* demuxer,
153 std::unique_ptr<AudioDecoderConfig> audio_config,
154 std::unique_ptr<VideoDecoderConfig> video_config,
155 MediaLog* media_log);
157 // Runs |read_cb_| if present with the front of |buffer_queue_|, calling
158 // NotifyCapacityAvailable() if capacity is still available.
159 void SatisfyPendingRead();
161 // Converts an FFmpeg stream timestamp into a base::TimeDelta.
162 static base::TimeDelta ConvertStreamTimestamp(const AVRational& time_base,
165 // Resets any currently active bitstream converter.
166 void ResetBitstreamConverter();
168 // Create new bitstream converter, destroying active converter if present.
169 void InitBitstreamConverter();
171 raw_ptr<FFmpegDemuxer> demuxer_;
172 scoped_refptr<base::SequencedTaskRunner> task_runner_;
173 raw_ptr<AVStream> stream_;
174 base::TimeDelta start_time_;
175 std::unique_ptr<AudioDecoderConfig> audio_config_;
176 std::unique_ptr<VideoDecoderConfig> video_config_;
177 raw_ptr<MediaLog> media_log_;
178 Type type_ = UNKNOWN;
179 StreamLiveness liveness_ = StreamLiveness::kUnknown;
180 base::TimeDelta duration_;
182 base::TimeDelta last_packet_timestamp_;
183 base::TimeDelta last_packet_duration_;
184 Ranges<base::TimeDelta> buffered_ranges_;
186 bool waiting_for_keyframe_;
189 DecoderBufferQueue buffer_queue_;
192 #if BUILDFLAG(USE_PROPRIETARY_CODECS)
193 std::unique_ptr<FFmpegBitstreamConverter> bitstream_converter_;
196 std::string encryption_key_id_;
197 bool fixup_negative_timestamps_;
198 bool fixup_chained_ogg_;
200 int num_discarded_packet_warnings_;
201 int64_t last_packet_pos_;
202 int64_t last_packet_dts_;
203 // Requested buffer count. The actual returned buffer count could be less
204 // according to DemuxerStream::Read() API.
205 size_t requested_buffer_count_ = 0;
208 class MEDIA_EXPORT FFmpegDemuxer : public Demuxer {
210 FFmpegDemuxer(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
211 DataSource* data_source,
212 const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
213 MediaTracksUpdatedCB media_tracks_updated_cb,
217 FFmpegDemuxer(const FFmpegDemuxer&) = delete;
218 FFmpegDemuxer& operator=(const FFmpegDemuxer&) = delete;
220 ~FFmpegDemuxer() override;
222 // Demuxer implementation.
223 std::string GetDisplayName() const override;
224 DemuxerType GetDemuxerType() const override;
225 void Initialize(DemuxerHost* host, PipelineStatusCallback init_cb) override;
226 void AbortPendingReads() override;
227 void Stop() override;
228 void StartWaitingForSeek(base::TimeDelta seek_time) override;
229 void CancelPendingSeek(base::TimeDelta seek_time) override;
230 void Seek(base::TimeDelta time, PipelineStatusCallback cb) override;
231 bool IsSeekable() const override;
232 base::Time GetTimelineOffset() const override;
233 std::vector<DemuxerStream*> GetAllStreams() override;
234 base::TimeDelta GetStartTime() const override;
235 int64_t GetMemoryUsage() const override;
236 absl::optional<container_names::MediaContainerName> GetContainerForMetrics()
239 // Calls |encrypted_media_init_data_cb_| with the initialization data
240 // encountered in the file.
241 void OnEncryptedMediaInitData(EmeInitDataType init_data_type,
242 const std::string& encryption_key_id);
244 // Allow FFmpegDemuxerStream to notify us when there is updated information
245 // about capacity and what buffered data is available.
246 void NotifyCapacityAvailable();
247 void NotifyBufferingChanged();
249 // Allow FFmpegDemxuerStream to notify us about an error.
250 void NotifyDemuxerError(PipelineStatus error);
252 void OnEnabledAudioTracksChanged(const std::vector<MediaTrack::Id>& track_ids,
253 base::TimeDelta curr_time,
254 TrackChangeCB change_completed_cb) override;
256 void OnSelectedVideoTrackChanged(const std::vector<MediaTrack::Id>& track_ids,
257 base::TimeDelta curr_time,
258 TrackChangeCB change_completed_cb) override;
259 void SetPlaybackRate(double rate) override {}
261 // The lowest demuxed timestamp. If negative, DemuxerStreams must use this to
262 // adjust packet timestamps such that external clients see a zero-based
264 base::TimeDelta start_time() const { return start_time_; }
266 // Task runner used to execute blocking FFmpeg operations.
267 scoped_refptr<base::SequencedTaskRunner> ffmpeg_task_runner() {
268 return blocking_task_runner_;
271 container_names::MediaContainerName container() const {
272 return glue_ ? glue_->container()
273 : container_names::MediaContainerName::kContainerUnknown;
277 // To allow tests access to privates.
278 friend class FFmpegDemuxerTest;
280 // Helper for vide and audio track changing.
281 void FindAndEnableProperTracks(const std::vector<MediaTrack::Id>& track_ids,
282 base::TimeDelta curr_time,
283 DemuxerStream::Type track_type,
284 TrackChangeCB change_completed_cb);
286 // FFmpeg callbacks during initialization.
287 void OnOpenContextDone(bool result);
288 void OnFindStreamInfoDone(int result);
290 void LogMetadata(AVFormatContext* avctx, base::TimeDelta max_duration);
292 // Finds the stream with the lowest known start time (i.e. not kNoTimestamp
293 // start time) with enabled status matching |enabled|.
294 FFmpegDemuxerStream* FindStreamWithLowestStartTimestamp(bool enabled);
296 // Finds a preferred stream for seeking to |seek_time|. Preference is
297 // typically given to video streams, unless the |seek_time| is earlier than
298 // the start time of the video stream. In that case a stream with the earliest
299 // start time is preferred. Disabled streams are considered only as the last
301 FFmpegDemuxerStream* FindPreferredStreamForSeeking(base::TimeDelta seek_time);
303 // FFmpeg callbacks during seeking.
304 void OnSeekFrameDone(int result);
306 // FFmpeg callbacks during reading + helper method to initiate reads.
307 void ReadFrameIfNeeded();
308 void OnReadFrameDone(ScopedAVPacket packet, int result);
310 // Returns true iff any stream has additional capacity. Note that streams can
311 // go over capacity depending on how the file is muxed.
312 bool StreamsHaveAvailableCapacity();
314 // Returns true if the maximum allowed memory usage has been reached.
315 bool IsMaxMemoryUsageReached() const;
317 // Signal all FFmpegDemuxerStreams that the stream has ended.
318 void StreamHasEnded();
320 // Called by |url_protocol_| whenever |data_source_| returns a read error.
321 void OnDataSourceError();
323 // Returns the first stream from |streams_| that matches |type| as an
324 // FFmpegDemuxerStream and is enabled.
325 FFmpegDemuxerStream* GetFirstEnabledFFmpegStream(
326 DemuxerStream::Type type) const;
328 void SetLiveness(StreamLiveness liveness);
330 void SeekInternal(base::TimeDelta time,
331 base::OnceCallback<void(int)> seek_cb);
332 void OnVideoSeekedForTrackChange(DemuxerStream* video_stream,
333 base::OnceClosure seek_completed_cb,
335 void SeekOnVideoTrackChange(base::TimeDelta seek_to_time,
336 TrackChangeCB seek_completed_cb,
337 DemuxerStream::Type stream_type,
338 const std::vector<DemuxerStream*>& streams);
340 // Executes |init_cb_| with |status| and closes out the async trace.
341 void RunInitCB(PipelineStatus status);
343 // Executes |pending_seek_cb_| with |status| and closes out the async trace.
344 void RunPendingSeekCB(PipelineStatus status);
346 raw_ptr<DemuxerHost, AcrossTasksDanglingUntriaged> host_ = nullptr;
348 scoped_refptr<base::SequencedTaskRunner> task_runner_;
350 // Task runner on which all blocking FFmpeg operations are executed; retrieved
351 // from base::ThreadPoolInstance.
352 scoped_refptr<base::SequencedTaskRunner> blocking_task_runner_;
354 PipelineStatusCallback init_cb_;
356 // Indicates if Stop() has been called.
357 bool stopped_ = false;
359 // Tracks if there's an outstanding av_read_frame() operation.
361 // TODO(scherkus): Allow more than one read in flight for higher read
362 // throughput using demuxer_bench to verify improvements.
363 bool pending_read_ = false;
365 // Tracks if there's an outstanding av_seek_frame() operation. Used to discard
366 // results of pre-seek av_read_frame() operations.
367 PipelineStatusCallback pending_seek_cb_;
369 // |streams_| mirrors the AVStream array in AVFormatContext. It contains
370 // FFmpegDemuxerStreams encapsluating AVStream objects at the same index.
372 // Since we only support a single audio and video stream, |streams_| will
373 // contain NULL entries for additional audio/video streams as well as for
374 // stream types that we do not currently support.
376 // Once initialized, operations on FFmpegDemuxerStreams should be carried out
377 // on the demuxer thread.
378 using StreamVector = std::vector<std::unique_ptr<FFmpegDemuxerStream>>;
379 StreamVector streams_;
381 // Provides asynchronous IO to this demuxer. Consumed by |url_protocol_| to
382 // integrate with libavformat.
383 raw_ptr<DataSource> data_source_;
385 raw_ptr<MediaLog> media_log_;
387 // Derived bitrate after initialization has completed.
390 // The first timestamp of the audio or video stream, whichever is lower. This
391 // is used to adjust timestamps so that external consumers always see a zero
393 base::TimeDelta start_time_ = kNoTimestamp;
395 // The Time associated with timestamp 0. Set to a null
396 // time if the file doesn't have an association to Time.
397 base::Time timeline_offset_;
399 // Set if we know duration of the audio stream. Used when processing end of
400 // stream -- at this moment we definitely know duration.
401 bool duration_known_ = false;
402 base::TimeDelta duration_;
404 // FFmpegURLProtocol implementation and corresponding glue bits.
405 std::unique_ptr<BlockingUrlProtocol> url_protocol_;
406 std::unique_ptr<FFmpegGlue> glue_;
408 const EncryptedMediaInitDataCB encrypted_media_init_data_cb_;
410 const MediaTracksUpdatedCB media_tracks_updated_cb_;
412 std::map<MediaTrack::Id, FFmpegDemuxerStream*> track_id_to_demux_stream_map_;
414 const bool is_local_file_;
416 // NOTE: Weak pointers must be invalidated before all other member variables.
417 base::WeakPtr<FFmpegDemuxer> weak_this_;
418 base::WeakPtrFactory<FFmpegDemuxer> cancel_pending_seek_factory_{this};
419 base::WeakPtrFactory<FFmpegDemuxer> weak_factory_{this};
424 #endif // MEDIA_FILTERS_FFMPEG_DEMUXER_H_