1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_RENDERERS_VIDEO_RENDERER_IMPL_H_
6 #define MEDIA_RENDERERS_VIDEO_RENDERER_IMPL_H_
13 #include "base/cancelable_callback.h"
14 #include "base/memory/raw_ptr.h"
15 #include "base/memory/scoped_refptr.h"
16 #include "base/memory/weak_ptr.h"
17 #include "base/synchronization/condition_variable.h"
18 #include "base/synchronization/lock.h"
19 #include "base/task/sequenced_task_runner.h"
20 #include "base/time/time.h"
21 #include "base/timer/timer.h"
22 #include "media/base/decryptor.h"
23 #include "media/base/demuxer_stream.h"
24 #include "media/base/frame_rate_estimator.h"
25 #include "media/base/limits.h"
26 #include "media/base/media_log.h"
27 #include "media/base/pipeline_status.h"
28 #include "media/base/tuneable.h"
29 #include "media/base/video_decoder.h"
30 #include "media/base/video_decoder_config.h"
31 #include "media/base/video_frame.h"
32 #include "media/base/video_renderer.h"
33 #include "media/base/video_renderer_sink.h"
34 #include "media/filters/decoder_stream.h"
35 #include "media/filters/video_renderer_algorithm.h"
36 #include "media/renderers/renderer_impl_factory.h"
37 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
45 // VideoRendererImpl handles reading from a VideoDecoderStream storing the
46 // results in a queue of decoded frames and executing a callback when a frame is
47 // ready for rendering.
48 class MEDIA_EXPORT VideoRendererImpl
49 : public VideoRenderer,
50 public VideoRendererSink::RenderCallback {
52 // |decoders| contains the VideoDecoders to use when initializing.
54 // Implementors should avoid doing any sort of heavy work in this method and
55 // instead post a task to a common/worker thread to handle rendering. Slowing
56 // down the video thread may result in losing synchronization with audio.
58 // Setting |drop_frames_| to true causes the renderer to drop expired frames.
60 const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
61 VideoRendererSink* sink,
62 const CreateVideoDecodersCB& create_video_decoders_cb,
65 std::unique_ptr<GpuMemoryBufferVideoFramePool> gmb_pool,
66 MediaPlayerLoggingID media_player_id);
68 VideoRendererImpl(const VideoRendererImpl&) = delete;
69 VideoRendererImpl& operator=(const VideoRendererImpl&) = delete;
71 ~VideoRendererImpl() override;
73 // VideoRenderer implementation.
74 void Initialize(DemuxerStream* stream,
75 CdmContext* cdm_context,
76 RendererClient* client,
77 const TimeSource::WallClockTimeCB& wall_clock_time_cb,
78 PipelineStatusCallback init_cb) override;
79 void Flush(base::OnceClosure callback) override;
80 void StartPlayingFrom(base::TimeDelta timestamp) override;
81 void OnTimeProgressing() override;
82 void OnTimeStopped() override;
83 void SetLatencyHint(absl::optional<base::TimeDelta> latency_hint) override;
85 void SetTickClockForTesting(const base::TickClock* tick_clock);
86 size_t frames_queued_for_testing() const {
87 return algorithm_->frames_queued();
89 size_t effective_frames_queued_for_testing() const {
90 return algorithm_->effective_frames_queued();
92 int min_buffered_frames_for_testing() const { return min_buffered_frames_; }
93 int max_buffered_frames_for_testing() const { return max_buffered_frames_; }
95 // VideoRendererSink::RenderCallback implementation.
96 scoped_refptr<VideoFrame> Render(base::TimeTicks deadline_min,
97 base::TimeTicks deadline_max,
98 RenderingMode rendering_mode) override;
99 void OnFrameDropped() override;
100 base::TimeDelta GetPreferredRenderInterval() override;
103 // Callback for |video_decoder_stream_| initialization.
104 void OnVideoDecoderStreamInitialized(bool success);
106 void FinishInitialization(PipelineStatus status);
109 // Functions to notify certain events to the RendererClient.
110 void OnPlaybackError(PipelineStatus error);
111 void OnPlaybackEnded();
112 void OnStatisticsUpdate(const PipelineStatistics& stats);
113 void OnBufferingStateChange(BufferingState state);
114 void OnWaiting(WaitingReason reason);
116 // Called by the VideoDecoderStream when a config change occurs. Will notify
117 // RenderClient of the new config.
118 void OnConfigChange(const VideoDecoderConfig& config);
120 // Called when the decoder stream and selector have a fallback after failed
122 void OnFallback(PipelineStatus status);
124 // Callback for |video_decoder_stream_| to deliver decoded video frames and
125 // report video decoding status.
126 void FrameReady(VideoDecoderStream::ReadResult result);
128 // Helper method for enqueueing a frame to |alogorithm_|.
129 void AddReadyFrame_Locked(scoped_refptr<VideoFrame> frame);
131 // Helper method that schedules an asynchronous read from the
132 // |video_decoder_stream_| as long as there isn't a pending read and we have
134 void AttemptRead_Locked();
136 // Called when VideoDecoderStream::Reset() completes.
137 void OnVideoDecoderStreamResetDone();
139 // Returns true if the renderer has enough data for playback purposes.
140 // Note that having enough data may be due to reaching end of stream.
141 bool HaveEnoughData_Locked() const;
142 void TransitionToHaveEnough_Locked();
143 void TransitionToHaveNothing();
144 void TransitionToHaveNothing_Locked();
146 // Runs |statistics_cb_| with |frames_decoded_| and |frames_dropped_|, resets
147 // them to 0. If |force_update| is true, sends an update even if no frames
148 // have been decoded since the last update.
149 void UpdateStats_Locked(bool force_update = false);
151 // Notifies |client_| if the current frame rate has changed since it was last
152 // reported, and tracks what the most recently reported frame rate was.
153 void ReportFrameRateIfNeeded_Locked();
155 // Update |min_buffered_frames_| and |max_buffered_frames_| using the latest
156 // |average_frame_duration|. Should only be called when |latency_hint_| > 0.
157 void UpdateLatencyHintBufferingCaps_Locked(
158 base::TimeDelta average_frame_duration);
160 // Returns true if algorithm_->effective_frames_queued() >= |buffering_cap|,
161 // or when the number of ineffective frames >= kAbsoluteMaxFrames.
162 bool HaveReachedBufferingCap(size_t buffering_cap) const;
164 // Starts or stops |sink_| respectively. Do not call while |lock_| is held.
168 // Fires |ended_cb_| if there are no remaining usable frames and
169 // |received_end_of_stream_| is true. Sets |rendered_end_of_stream_| if it
172 // When called from the media thread, |time_progressing| should reflect the
173 // value of |time_progressing_|. When called from Render() on the sink
174 // callback thread, |time_progressing| must be true since Render() could not
175 // have been called otherwise.
176 void MaybeFireEndedCallback_Locked(bool time_progressing);
178 // Helper method for converting a single media timestamp to wall clock time.
179 base::TimeTicks ConvertMediaTimestamp(base::TimeDelta media_timestamp);
181 base::TimeTicks GetCurrentMediaTimeAsWallClockTime();
183 // Helper method for checking if a frame timestamp plus the frame's expected
184 // duration is before |start_timestamp_|.
185 bool IsBeforeStartTime(const VideoFrame& frame);
187 // Helper method for checking if we have the best possible first frame to
188 // paint in the queue.
189 bool HasBestFirstFrame(const VideoFrame& frame);
191 // Attempts to remove frames which are no longer effective for rendering when
192 // |buffering_state_| == BUFFERING_HAVE_NOTHING or |was_background_rendering_|
193 // is true. If the current media time as provided by |wall_clock_time_cb_| is
194 // null, no frame expiration will be done.
196 // When background rendering the method will expire all frames before the
197 // current wall clock time since it's expected that there will be long delays
198 // between each Render() call in this state.
200 // When in the underflow state the method will first attempt to remove expired
201 // frames before the current media time plus duration. If |sink_started_| is
202 // true, nothing more can be done. However, if false, and there are still no
203 // effective frames in the queue, the entire frame queue will be released to
204 // avoid any stalling.
205 void RemoveFramesForUnderflowOrBackgroundRendering();
207 // Notifies |client_| in the event of frame size or opacity changes. Must be
208 // called on |task_runner_|.
209 void CheckForMetadataChanges(VideoPixelFormat pixel_format,
210 const gfx::Size& natural_size);
212 // Both calls AttemptRead_Locked() and CheckForMetadataChanges(). Must be
213 // called on |task_runner_|.
214 void AttemptReadAndCheckForMetadataChanges(VideoPixelFormat pixel_format,
215 const gfx::Size& natural_size);
217 // Paints first frame and sets `painted_first_frame_` to true if
218 // `painted_first_frame_` is false.
219 void PaintFirstFrame();
221 scoped_refptr<base::SequencedTaskRunner> task_runner_;
223 // Sink which calls into VideoRendererImpl via Render() for video frames. Do
224 // not call any methods on the sink while |lock_| is held or the two threads
225 // might deadlock. Do not call Start() or Stop() on the sink directly, use
226 // StartSink() and StopSink() to ensure background rendering is started. Only
227 // access these values on |task_runner_|.
228 const raw_ptr<VideoRendererSink, AcrossTasksDanglingUntriaged> sink_;
231 // Stores the last decoder config that was passed to
232 // RendererClient::OnVideoConfigChange. Used to prevent signaling config
233 // to the upper layers when when the new config is the same.
234 VideoDecoderConfig current_decoder_config_;
236 // Used for accessing data members.
239 raw_ptr<RendererClient> client_;
241 // Pool of GpuMemoryBuffers and resources used to create hardware frames.
242 // Ensure this is destructed after |algorithm_| for optimal memory release
243 // when a frames are still held by the compositor. Must be destructed after
244 // |video_decoder_stream_| since it holds a callback to the pool.
245 std::unique_ptr<GpuMemoryBufferVideoFramePool> gpu_memory_buffer_pool_;
247 // Provides video frames to VideoRendererImpl.
248 std::unique_ptr<VideoDecoderStream> video_decoder_stream_;
250 // Passed in during Initialize().
251 raw_ptr<DemuxerStream> demuxer_stream_;
253 // This dangling raw_ptr occurred in:
254 // webkit_unit_tests: WebMediaPlayerImplTest.MediaPositionState_Playing
255 // https://ci.chromium.org/ui/p/chromium/builders/try/linux-rel/1425143/test-results?q=ExactID%3Aninja%3A%2F%2Fthird_party%2Fblink%2Frenderer%2Fcontroller%3Ablink_unittests%2FWebMediaPlayerImplTest.MediaPositionState_Playing+VHash%3A896f1103f2d1008d&sortby=&groupby=
256 raw_ptr<MediaLog, FlakyDanglingUntriaged> media_log_;
258 MediaPlayerLoggingID player_id_;
260 // Flag indicating low-delay mode.
263 // Keeps track of whether we received the end of stream buffer and finished
265 bool received_end_of_stream_;
266 bool rendered_end_of_stream_;
268 // Important detail: being in kPlaying doesn't imply that video is being
269 // rendered. Rather, it means that the renderer is ready to go. The actual
270 // rendering of video is controlled by time advancing via |get_time_cb_|.
271 // Video renderer can be reinitialized completely by calling Initialize again
272 // when it is in a kFlushed state with video sink stopped.
275 // +------> | Initialize()
279 // | | Decoders initialized
281 // | V Decoders reset
282 // ---- kFlushed <------------------ kFlushing
283 // | StartPlayingFrom() ^
286 // `---------> kPlaying --------'
287 enum State { kUninitialized, kInitializing, kFlushing, kFlushed, kPlaying };
290 // TODO(servolk): Consider using DecoderFactory here instead of the
291 // CreateVideoDecodersCB.
292 CreateVideoDecodersCB create_video_decoders_cb_;
294 // Keep track of the outstanding read on the VideoDecoderStream. Flushing can
295 // only complete once the read has completed.
300 BufferingState buffering_state_;
302 // Playback operation callbacks.
303 PipelineStatusCallback init_cb_;
304 base::OnceClosure flush_cb_;
305 TimeSource::WallClockTimeCB wall_clock_time_cb_;
307 base::TimeDelta start_timestamp_;
309 // Keeps track of the number of frames decoded and dropped since the
310 // last call to |statistics_cb_|. These must be accessed under lock.
311 PipelineStatistics stats_;
313 raw_ptr<const base::TickClock> tick_clock_;
315 // Algorithm for selecting which frame to render; manages frames and all
316 // timing related information. Ensure this is destructed before
317 // |gpu_memory_buffer_pool_| for optimal memory release when a frames are
318 // still held by the compositor.
319 std::unique_ptr<VideoRendererAlgorithm> algorithm_;
321 // Indicates that Render() was called with |background_rendering| set to true,
322 // so we've entered a background rendering mode where dropped frames are not
323 // counted. Must be accessed under |lock_| once |sink_| is started.
324 bool was_background_rendering_;
326 // Indicates whether or not media time is currently progressing or not. Must
327 // only be accessed from |task_runner_|.
328 bool time_progressing_;
330 // Indicates if a frame has been processed by CheckForMetadataChanges().
331 bool have_renderered_frames_;
333 // Tracks last frame properties to detect and notify client of any changes.
334 gfx::Size last_frame_natural_size_;
335 bool last_frame_opaque_;
337 // The last value from |video_decoder_stream_->AverageDuration()|.
338 base::TimeDelta last_decoder_stream_avg_duration_;
340 // Indicates if we've painted the first valid frame after StartPlayingFrom().
341 bool painted_first_frame_;
343 // Used to paint the first frame if we don't receive the best one in time and
344 // aren't guaranteed to receive anymore.
345 base::CancelableOnceClosure paint_first_frame_cb_;
347 // The initial value for |min_buffered_frames_| and |max_buffered_frames_|.
348 Tuneable<size_t> initial_buffering_size_ = {
349 "MediaInitialBufferingSizeForHaveEnough", 3, limits::kMaxVideoFrames, 10};
351 // The number of frames required to transition from BUFFERING_HAVE_NOTHING to
352 // BUFFERING_HAVE_ENOUGH.
353 size_t min_buffered_frames_;
355 // The maximum number of frames to buffer. Always >= |min_buffered_frames_|.
356 // May be greater-than when |latency_hint_| set that decreases the minimum
358 size_t max_buffered_frames_;
360 // Last Render() and last FrameReady() times respectively. Used to avoid
361 // triggering underflow when background rendering.
362 base::TimeTicks last_render_time_;
363 base::TimeTicks last_frame_ready_time_;
365 // Running average of frame durations.
366 FrameRateEstimator fps_estimator_;
368 // Last FPS, if any, reported to the client.
369 absl::optional<int> last_reported_fps_;
371 // Value saved from last call to SetLatencyHint(). Used to recompute buffering
372 // limits as framerate fluctuates.
373 absl::optional<base::TimeDelta> latency_hint_;
375 // When latency_hint_ > 0, we make regular adjustments to buffering caps as
376 // |algorithm_->average_frame_duration()| fluctuates, but we only want to emit
378 bool is_latency_hint_media_logged_ = false;
380 // NOTE: Weak pointers must be invalidated before all other member variables.
381 base::WeakPtrFactory<VideoRendererImpl> weak_factory_{this};
383 // Weak factory used to invalidate certain queued callbacks on reset().
384 // This is useful when doing video frame copies asynchronously since we
385 // want to discard video frames that might be received after the stream has
387 base::WeakPtrFactory<VideoRendererImpl> cancel_on_flush_weak_factory_{this};
392 #endif // MEDIA_RENDERERS_VIDEO_RENDERER_IMPL_H_