1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/renderers/video_renderer_impl.h"
11 #include "base/feature_list.h"
12 #include "base/functional/bind.h"
13 #include "base/functional/callback.h"
14 #include "base/functional/callback_helpers.h"
15 #include "base/location.h"
16 #include "base/metrics/histogram_macros.h"
17 #include "base/numerics/safe_conversions.h"
18 #include "base/strings/string_util.h"
19 #include "base/task/bind_post_task.h"
20 #include "base/task/sequenced_task_runner.h"
21 #include "base/task/single_thread_task_runner.h"
22 #include "base/time/default_tick_clock.h"
23 #include "base/trace_event/trace_event.h"
24 #include "media/base/media_log.h"
25 #include "media/base/media_switches.h"
26 #include "media/base/pipeline_status.h"
27 #include "media/base/renderer_client.h"
28 #include "media/base/video_frame.h"
34 // Maximum number of frames we will buffer, regardless of their "effectiveness".
35 // See HaveReachedBufferingCap(). The value was historically described in terms
36 // of |min_buffered_frames_| as follows:
37 // = 3 * high_water_mark(min_buffered_frames_),
38 // = 3 * (2 * limits::kMaxVideoFrames)
40 // Today, |min_buffered_frames_| can go down (as low as 1) and up in response to
41 // SetLatencyHint(), so we needed to peg this with a constant.
42 constexpr int kAbsoluteMaxFrames = 24;
44 bool ShouldUseLowDelayMode(DemuxerStream* stream) {
45 return base::FeatureList::IsEnabled(kLowDelayVideoRenderingOnLiveStream) &&
46 stream->liveness() == StreamLiveness::kLive;
51 VideoRendererImpl::VideoRendererImpl(
52 const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
53 VideoRendererSink* sink,
54 const CreateVideoDecodersCB& create_video_decoders_cb,
57 std::unique_ptr<GpuMemoryBufferVideoFramePool> gmb_pool,
58 MediaPlayerLoggingID media_player_id)
59 : task_runner_(media_task_runner),
63 gpu_memory_buffer_pool_(std::move(gmb_pool)),
64 media_log_(media_log),
65 player_id_(media_player_id),
67 received_end_of_stream_(false),
68 rendered_end_of_stream_(false),
69 state_(kUninitialized),
70 create_video_decoders_cb_(create_video_decoders_cb),
72 drop_frames_(drop_frames),
73 buffering_state_(BUFFERING_HAVE_NOTHING),
74 tick_clock_(base::DefaultTickClock::GetInstance()),
75 was_background_rendering_(false),
76 time_progressing_(false),
77 have_renderered_frames_(false),
78 last_frame_opaque_(false),
79 painted_first_frame_(false),
80 min_buffered_frames_(initial_buffering_size_.value()),
81 max_buffered_frames_(initial_buffering_size_.value()) {
82 DCHECK(create_video_decoders_cb_);
85 VideoRendererImpl::~VideoRendererImpl() {
86 DCHECK(task_runner_->RunsTasksInCurrentSequence());
89 FinishInitialization(PIPELINE_ERROR_ABORT);
98 void VideoRendererImpl::Flush(base::OnceClosure callback) {
100 DCHECK(task_runner_->RunsTasksInCurrentSequence());
105 base::AutoLock auto_lock(lock_);
107 DCHECK_EQ(state_, kPlaying);
108 flush_cb_ = std::move(callback);
111 if (buffering_state_ != BUFFERING_HAVE_NOTHING) {
112 buffering_state_ = BUFFERING_HAVE_NOTHING;
113 task_runner_->PostTask(
115 base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
116 weak_factory_.GetWeakPtr(), buffering_state_));
118 received_end_of_stream_ = false;
119 rendered_end_of_stream_ = false;
121 // Reset |video_decoder_stream_| and drop any pending read callbacks from it.
122 pending_read_ = false;
123 if (gpu_memory_buffer_pool_)
124 gpu_memory_buffer_pool_->Abort();
125 cancel_on_flush_weak_factory_.InvalidateWeakPtrs();
126 paint_first_frame_cb_.Cancel();
127 video_decoder_stream_->Reset(
128 base::BindOnce(&VideoRendererImpl::OnVideoDecoderStreamResetDone,
129 weak_factory_.GetWeakPtr()));
131 // To avoid unnecessary work by VDAs, only delete queued frames after
132 // resetting |video_decoder_stream_|. If this is done in the opposite order
133 // VDAs will get a bunch of ReusePictureBuffer() calls before the Reset(),
134 // which they may use to output more frames that won't be used.
136 painted_first_frame_ = false;
138 // Reset preroll capacity so seek time is not penalized. |latency_hint_|
139 // and |low_delay_| mode disable automatic preroll adjustments.
140 if (!latency_hint_.has_value() && !low_delay_) {
141 min_buffered_frames_ = max_buffered_frames_ =
142 initial_buffering_size_.value();
146 void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) {
147 DVLOG(1) << __func__ << "(" << timestamp.InMicroseconds() << ")";
148 DCHECK(task_runner_->RunsTasksInCurrentSequence());
149 base::AutoLock auto_lock(lock_);
150 DCHECK_EQ(state_, kFlushed);
151 DCHECK(!pending_read_);
152 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
155 start_timestamp_ = timestamp;
156 painted_first_frame_ = false;
157 last_render_time_ = last_frame_ready_time_ = base::TimeTicks();
158 video_decoder_stream_->SkipPrepareUntil(start_timestamp_);
159 AttemptRead_Locked();
162 void VideoRendererImpl::Initialize(
163 DemuxerStream* stream,
164 CdmContext* cdm_context,
165 RendererClient* client,
166 const TimeSource::WallClockTimeCB& wall_clock_time_cb,
167 PipelineStatusCallback init_cb) {
168 DCHECK(task_runner_->RunsTasksInCurrentSequence());
169 TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "VideoRendererImpl::Initialize",
170 TRACE_ID_LOCAL(this));
172 base::AutoLock auto_lock(lock_);
174 DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
176 DCHECK(wall_clock_time_cb);
177 DCHECK(kUninitialized == state_ || kFlushed == state_);
178 DCHECK(!was_background_rendering_);
179 DCHECK(!time_progressing_);
181 demuxer_stream_ = stream;
183 video_decoder_stream_ = std::make_unique<VideoDecoderStream>(
184 std::make_unique<VideoDecoderStream::StreamTraits>(media_log_),
185 task_runner_, create_video_decoders_cb_, media_log_);
186 video_decoder_stream_->set_config_change_observer(base::BindRepeating(
187 &VideoRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
188 video_decoder_stream_->set_fallback_observer(base::BindRepeating(
189 &VideoRendererImpl::OnFallback, weak_factory_.GetWeakPtr()));
190 if (gpu_memory_buffer_pool_) {
191 video_decoder_stream_->SetPrepareCB(base::BindRepeating(
192 &GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame,
193 // Safe since VideoDecoderStream won't issue calls after destruction.
194 base::Unretained(gpu_memory_buffer_pool_.get())));
197 low_delay_ = ShouldUseLowDelayMode(demuxer_stream_);
199 MEDIA_LOG(DEBUG, media_log_) << "Video rendering in low delay mode.";
201 // "Low delay mode" means only one frame must be buffered to transition to
202 // BUFFERING_HAVE_ENOUGH.
203 min_buffered_frames_ = 1;
206 // Always post |init_cb_| because |this| could be destroyed if initialization
208 init_cb_ = base::BindPostTaskToCurrentDefault(std::move(init_cb));
211 wall_clock_time_cb_ = wall_clock_time_cb;
212 state_ = kInitializing;
214 current_decoder_config_ = demuxer_stream_->video_decoder_config();
215 DCHECK(current_decoder_config_.IsValidConfig());
217 video_decoder_stream_->Initialize(
219 base::BindOnce(&VideoRendererImpl::OnVideoDecoderStreamInitialized,
220 weak_factory_.GetWeakPtr()),
222 base::BindRepeating(&VideoRendererImpl::OnStatisticsUpdate,
223 weak_factory_.GetWeakPtr()),
224 base::BindRepeating(&VideoRendererImpl::OnWaiting,
225 weak_factory_.GetWeakPtr()));
228 scoped_refptr<VideoFrame> VideoRendererImpl::Render(
229 base::TimeTicks deadline_min,
230 base::TimeTicks deadline_max,
231 RenderingMode rendering_mode) {
232 TRACE_EVENT_BEGIN1("media", "VideoRendererImpl::Render", "id", player_id_);
233 base::AutoLock auto_lock(lock_);
234 DCHECK_EQ(state_, kPlaying);
235 last_render_time_ = tick_clock_->NowTicks();
237 size_t frames_dropped = 0;
238 scoped_refptr<VideoFrame> result =
239 algorithm_->Render(deadline_min, deadline_max, &frames_dropped);
241 // Due to how the |algorithm_| holds frames, this should never be null if
242 // we've had a proper startup sequence.
245 const bool background_rendering =
246 rendering_mode == RenderingMode::kBackground;
248 // Declare HAVE_NOTHING if we reach a state where we can't progress playback
249 // any further. We don't want to do this if we've already done so, reached
250 // end of stream, or have frames available. We also don't want to do this in
251 // background rendering mode, as the frames aren't visible anyways.
252 MaybeFireEndedCallback_Locked(true);
253 if (buffering_state_ == BUFFERING_HAVE_ENOUGH && !received_end_of_stream_ &&
254 !algorithm_->effective_frames_queued() && !background_rendering &&
255 !was_background_rendering_) {
256 // Do not set |buffering_state_| here as the lock in FrameReady() may be
257 // held already and it fire the state changes in the wrong order.
258 DVLOG(3) << __func__ << " posted TransitionToHaveNothing.";
259 task_runner_->PostTask(
260 FROM_HERE, base::BindOnce(&VideoRendererImpl::TransitionToHaveNothing,
261 weak_factory_.GetWeakPtr()));
264 // We don't count dropped frames in the background to avoid skewing the count
265 // and impacting JavaScript visible metrics used by web developers.
267 // Just after resuming from background rendering, we also don't count the
268 // dropped frames since they are likely just dropped due to being too old.
269 if (!background_rendering && !was_background_rendering_)
270 stats_.video_frames_dropped += frames_dropped;
271 was_background_rendering_ = background_rendering;
273 // Always post this task, it will acquire new frames if necessary and since it
274 // happens on another thread, even if we don't have room in the queue now, by
275 // the time it runs (may be delayed up to 50ms for complex decodes!) we might.
276 task_runner_->PostTask(
278 base::BindOnce(&VideoRendererImpl::AttemptReadAndCheckForMetadataChanges,
279 weak_factory_.GetWeakPtr(), result->format(),
280 result->natural_size()));
282 TRACE_EVENT_END1("media", "VideoRendererImpl::Render", "frame",
283 result->AsHumanReadableString());
287 void VideoRendererImpl::OnFrameDropped() {
288 base::AutoLock auto_lock(lock_);
289 algorithm_->OnLastFrameDropped();
292 base::TimeDelta VideoRendererImpl::GetPreferredRenderInterval() {
293 base::AutoLock auto_lock(lock_);
294 return algorithm_->average_frame_duration();
297 void VideoRendererImpl::OnVideoDecoderStreamInitialized(bool success) {
298 DCHECK(task_runner_->RunsTasksInCurrentSequence());
299 base::AutoLock auto_lock(lock_);
300 DCHECK_EQ(state_, kInitializing);
303 state_ = kUninitialized;
304 FinishInitialization(DECODER_ERROR_NOT_SUPPORTED);
308 // We're all good! Consider ourselves flushed because we have not read any
313 std::make_unique<VideoRendererAlgorithm>(wall_clock_time_cb_, media_log_);
315 algorithm_->disable_frame_dropping();
317 FinishInitialization(PIPELINE_OK);
320 void VideoRendererImpl::FinishInitialization(PipelineStatus status) {
322 TRACE_EVENT_NESTABLE_ASYNC_END1("media", "VideoRendererImpl::Initialize",
323 TRACE_ID_LOCAL(this), "status",
324 PipelineStatusToString(status));
325 std::move(init_cb_).Run(status);
328 void VideoRendererImpl::FinishFlush() {
330 TRACE_EVENT_NESTABLE_ASYNC_END0("media", "VideoRendererImpl::Flush",
331 TRACE_ID_LOCAL(this));
332 std::move(flush_cb_).Run();
335 void VideoRendererImpl::OnPlaybackError(PipelineStatus error) {
336 DCHECK(task_runner_->RunsTasksInCurrentSequence());
337 client_->OnError(error);
340 void VideoRendererImpl::OnPlaybackEnded() {
341 DCHECK(task_runner_->RunsTasksInCurrentSequence());
343 // Send one last stats update so things like memory usage are correct.
344 base::AutoLock auto_lock(lock_);
345 UpdateStats_Locked(true);
351 void VideoRendererImpl::OnStatisticsUpdate(const PipelineStatistics& stats) {
352 DCHECK(task_runner_->RunsTasksInCurrentSequence());
353 client_->OnStatisticsUpdate(stats);
356 void VideoRendererImpl::OnBufferingStateChange(BufferingState buffering_state) {
357 DCHECK(task_runner_->RunsTasksInCurrentSequence());
359 // "Underflow" is only possible when playing. This avoids noise like blaming
360 // the decoder for an "underflow" that is really just a seek.
361 BufferingStateChangeReason reason = BUFFERING_CHANGE_REASON_UNKNOWN;
362 if (state_ == kPlaying && buffering_state == BUFFERING_HAVE_NOTHING) {
363 reason = video_decoder_stream_->is_demuxer_read_pending()
368 media_log_->AddEvent<MediaLogEvent::kBufferingStateChanged>(
369 SerializableBufferingState<SerializableBufferingStateType::kVideo>{
370 buffering_state, reason});
372 client_->OnBufferingStateChange(buffering_state, reason);
375 void VideoRendererImpl::OnWaiting(WaitingReason reason) {
376 DCHECK(task_runner_->RunsTasksInCurrentSequence());
377 client_->OnWaiting(reason);
380 void VideoRendererImpl::OnConfigChange(const VideoDecoderConfig& config) {
381 DCHECK(task_runner_->RunsTasksInCurrentSequence());
382 DCHECK(config.IsValidConfig());
384 // RendererClient only cares to know about config changes that differ from
386 if (!current_decoder_config_.Matches(config)) {
387 current_decoder_config_ = config;
388 client_->OnVideoConfigChange(config);
392 void VideoRendererImpl::OnFallback(PipelineStatus status) {
393 DCHECK(task_runner_->RunsTasksInCurrentSequence());
394 client_->OnFallback(std::move(status).AddHere());
397 void VideoRendererImpl::SetTickClockForTesting(
398 const base::TickClock* tick_clock) {
399 tick_clock_ = tick_clock;
402 void VideoRendererImpl::OnTimeProgressing() {
403 DCHECK(task_runner_->RunsTasksInCurrentSequence());
405 // WARNING: Do not attempt to use |lock_| here as StartSink() may cause a
408 time_progressing_ = true;
413 // If only an EOS frame came in after a seek, the renderer may not have
414 // received the ended event yet though we've posted it.
415 if (rendered_end_of_stream_)
418 // If we have no frames queued, there is a pending buffering state change in
419 // flight and we should ignore the start attempt.
420 if (!algorithm_->frames_queued()) {
421 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
428 void VideoRendererImpl::OnTimeStopped() {
429 DCHECK(task_runner_->RunsTasksInCurrentSequence());
431 // WARNING: Do not attempt to use |lock_| here as StopSink() may cause a
434 time_progressing_ = false;
441 // Make sure we expire everything we can if we can't read any more currently,
442 // otherwise playback may hang indefinitely. Note: There are no effective
443 // frames queued at this point, otherwise FrameReady() would have canceled
444 // the underflow state before reaching this point.
445 if (buffering_state_ == BUFFERING_HAVE_NOTHING) {
446 base::AutoLock al(lock_);
447 RemoveFramesForUnderflowOrBackgroundRendering();
449 // If we've underflowed, increase the number of frames required to reach
450 // BUFFERING_HAVE_ENOUGH upon resume; this will help prevent us from
451 // repeatedly underflowing. Providing a |latency_hint_| or enabling
452 // |low_delay_| mode disables automatic increases. In these cases the site
453 // is expressing a desire to manually control/minimize the buffering
454 // threshold for HAVE_ENOUGH.
455 const size_t kMaxUnderflowGrowth = 2 * initial_buffering_size_.value();
456 if (!latency_hint_.has_value() && !low_delay_) {
457 DCHECK_EQ(min_buffered_frames_, max_buffered_frames_);
459 if (min_buffered_frames_ < kMaxUnderflowGrowth) {
460 min_buffered_frames_++;
461 DVLOG(2) << __func__ << " Underflow! Increased min_buffered_frames_: "
462 << min_buffered_frames_;
466 // Increase |max_buffered_frames_| irrespective of |latency_hint_| and
467 // |low_delay_| mode. Unlike |min_buffered_frames_|, this does not affect
468 // the buffering threshold for HAVE_ENOUGH. When max > min, the renderer can
469 // buffer frames _beyond_ the HAVE_ENOUGH threshold (assuming decoder is
470 // fast enough), which still helps reduce the likelihood of repeat
472 if (max_buffered_frames_ < kMaxUnderflowGrowth) {
473 max_buffered_frames_++;
474 DVLOG(2) << __func__ << " Underflow! Increased max_buffered_frames_: "
475 << max_buffered_frames_;
480 void VideoRendererImpl::SetLatencyHint(
481 absl::optional<base::TimeDelta> latency_hint) {
482 base::AutoLock auto_lock(lock_);
484 latency_hint_ = latency_hint;
486 // Permanently disable implicit |low_delay_| mode. Apps using latencyHint
487 // are taking manual control of how buffering works. Unsetting the hint
488 // will make rendering behave as if |low_delay_| were never set.
491 if (!latency_hint_.has_value()) {
492 // Restore default values.
493 // NOTE |initial_buffering_size_| the default max, not the max overall.
494 min_buffered_frames_ = max_buffered_frames_ =
495 initial_buffering_size_.value();
496 MEDIA_LOG(DEBUG, media_log_)
497 << "Video latency hint cleared. Default buffer size ("
498 << min_buffered_frames_ << " frames) restored";
499 } else if (latency_hint_->is_zero()) {
500 // Zero is a special case implying the bare minimum buffering (1 frame).
501 // We apply the hint here outside of UpdateLatencyHintBufferingCaps_Locked()
502 // to avoid needless churn since the "bare minimum" buffering doesn't
503 // fluctuate with changes to FPS.
504 min_buffered_frames_ = 1;
505 max_buffered_frames_ = initial_buffering_size_.value();
506 MEDIA_LOG(DEBUG, media_log_)
507 << "Video latency hint set:" << *latency_hint << ". "
508 << "Effective buffering latency: 1 frame";
510 // Non-zero latency hints are set here. Update buffering caps immediately if
511 // we already have an algorithm_. Otherwise, the update will be applied as
512 // frames arrive and duration becomes known. The caps will be recalculated
513 // for each frame in case |average_frame_druation| changes.
514 // |is_latency_hint_media_logged_| ensures that we only MEDIA_LOG on the
515 // first application of this hint.
516 is_latency_hint_media_logged_ = false;
518 UpdateLatencyHintBufferingCaps_Locked(
519 algorithm_->average_frame_duration());
524 void VideoRendererImpl::UpdateLatencyHintBufferingCaps_Locked(
525 base::TimeDelta average_frame_duration) {
526 lock_.AssertAcquired();
528 // NOTE: this method may be called for every frame. Only perform trivial
531 // This method should only be called for non-zero latency hints. Zero is hard
532 // coded to 1 frame inside SetLatencyHint().
533 DCHECK(latency_hint_.has_value() && !latency_hint_->is_zero());
535 // For hints > 0, we need |average_frame_duration| to determine how many
536 // frames would yield the specified target latency. This method will be called
537 // again as |average_frame_duration| changes.
538 if (average_frame_duration.is_zero())
541 int latency_hint_frames =
542 base::ClampRound(*latency_hint_ / average_frame_duration);
544 std::string clamp_string;
545 if (latency_hint_frames > kAbsoluteMaxFrames) {
546 min_buffered_frames_ = kAbsoluteMaxFrames;
547 clamp_string = " (clamped to max)";
548 } else if (latency_hint_frames < 1) {
549 min_buffered_frames_ = 1;
550 clamp_string = " (clamped to min)";
552 min_buffered_frames_ = latency_hint_frames;
555 // Use initial capacity limit if possible. Increase if needed.
556 max_buffered_frames_ =
557 std::max(min_buffered_frames_, initial_buffering_size_.value());
559 if (!is_latency_hint_media_logged_) {
560 is_latency_hint_media_logged_ = true;
561 MEDIA_LOG(DEBUG, media_log_)
562 << "Video latency hint set:" << *latency_hint_ << ". "
563 << "Effective buffering latency:"
564 << (min_buffered_frames_ * average_frame_duration) << clamp_string;
568 void VideoRendererImpl::FrameReady(VideoDecoderStream::ReadResult result) {
569 DCHECK(task_runner_->RunsTasksInCurrentSequence());
570 base::AutoLock auto_lock(lock_);
571 DCHECK_EQ(state_, kPlaying);
572 CHECK(pending_read_);
573 pending_read_ = false;
575 // Can happen when demuxers are preparing for a new Seek().
576 switch (result.code()) {
577 case DecoderStatus::Codes::kOk:
579 case DecoderStatus::Codes::kAborted:
580 // TODO(liberato): This used to check specifically for the value
581 // DEMUXER_READ_ABORTED, which was more specific than |kAborted|.
582 // However, since it's a dcheck, this seems okay.
585 // Anything other than `kOk` or `kAborted` is treated as an error.
586 DCHECK(!result.has_value());
588 PipelineStatus::Codes code =
589 result.code() == DecoderStatus::Codes::kDisconnected
590 ? PIPELINE_ERROR_DISCONNECTED
591 : PIPELINE_ERROR_DECODE;
592 PipelineStatus status = {code, std::move(result).error()};
593 task_runner_->PostTask(
595 base::BindOnce(&VideoRendererImpl::OnPlaybackError,
596 weak_factory_.GetWeakPtr(), std::move(status)));
600 DCHECK(result.has_value());
601 scoped_refptr<VideoFrame> frame = std::move(result).value();
604 last_frame_ready_time_ = tick_clock_->NowTicks();
605 last_decoder_stream_avg_duration_ = video_decoder_stream_->AverageDuration();
607 const bool is_eos = frame->metadata().end_of_stream;
608 const bool is_before_start_time = !is_eos && IsBeforeStartTime(*frame);
609 const bool cant_read = !video_decoder_stream_->CanReadWithoutStalling();
610 const bool has_best_first_frame = !is_eos && HasBestFirstFrame(*frame);
611 const auto format = frame->format();
612 const auto natural_size = frame->natural_size();
615 DCHECK(!received_end_of_stream_);
616 received_end_of_stream_ = true;
617 fps_estimator_.Reset();
618 ReportFrameRateIfNeeded_Locked();
619 } else if ((min_buffered_frames_ == 1 || cant_read) && is_before_start_time) {
620 // Don't accumulate frames that are earlier than the start time if we
621 // won't have a chance for a better frame, otherwise we could declare
622 // HAVE_ENOUGH_DATA and start playback prematurely.
623 fps_estimator_.Reset();
624 ReportFrameRateIfNeeded_Locked();
625 AttemptRead_Locked();
628 // If the sink hasn't been started, we still have time to release less
629 // than ideal frames prior to startup. We don't use IsBeforeStartTime()
630 // here since it's based on a duration estimate and we can be exact here.
631 if (!sink_started_ && frame->timestamp() <= start_timestamp_) {
633 fps_estimator_.Reset();
634 ReportFrameRateIfNeeded_Locked();
637 // Provide frame duration information so that even if we only have one frame
638 // in the queue we can properly estimate duration. This allows the call to
639 // RemoveFramesForUnderflowOrBackgroundRendering() below to actually expire
640 // this frame if it's too far behind the current media time. Without this,
641 // we may resume too soon after a track change in the low delay case.
642 if (!frame->metadata().frame_duration.has_value())
643 frame->metadata().frame_duration = last_decoder_stream_avg_duration_;
645 AddReadyFrame_Locked(std::move(frame));
648 // Attempt to purge bad frames in case of underflow or backgrounding.
649 RemoveFramesForUnderflowOrBackgroundRendering();
651 // Paint the first frame if possible and necessary. Paint ahead of
652 // HAVE_ENOUGH_DATA to ensure the user sees the frame as early as possible.
653 // Paint before calling algorithm_->average_frame_duration(), as the call to
654 // Render() will trigger internal duration updates.
656 // We want to paint the first frame under two conditions: Either (1) we have
657 // enough frames to know it's definitely the first frame or (2) there may be
658 // no more frames coming (sometimes unless we paint one of them).
660 // We have to check both effective_frames_queued() and |has_best_first_frame|
661 // since prior to the clock starting effective_frames_queued() is a guess.
663 // NOTE: Do this before using algorithm_->average_frame_duration(). This
664 // initial render will update the duration to be non-zero when provided by
666 if (!sink_started_ && !painted_first_frame_ && algorithm_->frames_queued()) {
667 if (received_end_of_stream_ ||
668 (algorithm_->effective_frames_queued() && has_best_first_frame)) {
670 } else if (cant_read) {
671 // `cant_read` isn't always reliable, so only paint after 250ms if we
672 // haven't gotten anything better. This resets for each frame received. We
673 // still kick off any metadata changes to avoid any layout shift though.
674 CheckForMetadataChanges(format, natural_size);
675 paint_first_frame_cb_.Reset(base::BindOnce(
676 &VideoRendererImpl::PaintFirstFrame, base::Unretained(this)));
677 task_runner_->PostDelayedTask(FROM_HERE, paint_first_frame_cb_.callback(),
678 base::Milliseconds(250));
682 // Update average frame duration.
683 base::TimeDelta frame_duration = algorithm_->average_frame_duration();
684 if (frame_duration != kNoTimestamp && frame_duration != base::Seconds(0)) {
685 fps_estimator_.AddSample(frame_duration);
687 fps_estimator_.Reset();
689 ReportFrameRateIfNeeded_Locked();
691 // Update any statistics since the last call.
692 UpdateStats_Locked();
694 // Update hint-driven buffering caps to use the latest average frame duration.
695 // NOTE: Do this before updating the buffering state below, as it may affect
696 // the outcome of HaveEnoughData_Locked().
697 // TODO(chcunningham): Duration from |algorithm_| is affected by playback
698 // rate. Consider using wall clock frame duration instead.
699 if (latency_hint_.has_value() && !latency_hint_->is_zero())
700 UpdateLatencyHintBufferingCaps_Locked(frame_duration);
702 // Signal buffering state if we've met our conditions.
703 if (buffering_state_ == BUFFERING_HAVE_NOTHING && HaveEnoughData_Locked())
704 TransitionToHaveEnough_Locked();
706 // We may have removed all frames above and have reached end of stream. This
707 // must happen after the buffering state change has been signaled.
708 MaybeFireEndedCallback_Locked(time_progressing_);
710 // Always request more decoded video if we have capacity.
711 AttemptRead_Locked();
714 bool VideoRendererImpl::HaveEnoughData_Locked() const {
715 DCHECK_EQ(state_, kPlaying);
716 lock_.AssertAcquired();
718 if (received_end_of_stream_)
721 if (HaveReachedBufferingCap(min_buffered_frames_))
724 // If we've decoded any frames since the last render, signal have enough to
725 // avoid underflowing when video is not visible unless we run out of frames.
726 if (was_background_rendering_ && last_frame_ready_time_ >= last_render_time_)
729 if (min_buffered_frames_ > 1 &&
730 video_decoder_stream_->CanReadWithoutStalling()) {
734 // Note: We still require an effective frame in the stalling case since this
735 // method is also used to inform TransitionToHaveNothing_Locked() and thus
736 // would never pause and rebuffer if we always return true here.
737 return algorithm_->effective_frames_queued() > 0u;
740 void VideoRendererImpl::TransitionToHaveEnough_Locked() {
741 DVLOG(3) << __func__;
742 DCHECK(task_runner_->RunsTasksInCurrentSequence());
743 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
744 lock_.AssertAcquired();
746 buffering_state_ = BUFFERING_HAVE_ENOUGH;
747 task_runner_->PostTask(
748 FROM_HERE, base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
749 weak_factory_.GetWeakPtr(), buffering_state_));
752 void VideoRendererImpl::TransitionToHaveNothing() {
753 DVLOG(3) << __func__;
754 DCHECK(task_runner_->RunsTasksInCurrentSequence());
756 base::AutoLock auto_lock(lock_);
757 TransitionToHaveNothing_Locked();
760 void VideoRendererImpl::TransitionToHaveNothing_Locked() {
761 DVLOG(3) << __func__;
762 DCHECK(task_runner_->RunsTasksInCurrentSequence());
763 lock_.AssertAcquired();
765 if (buffering_state_ != BUFFERING_HAVE_ENOUGH || HaveEnoughData_Locked())
768 buffering_state_ = BUFFERING_HAVE_NOTHING;
769 task_runner_->PostTask(
770 FROM_HERE, base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
771 weak_factory_.GetWeakPtr(), buffering_state_));
774 void VideoRendererImpl::AddReadyFrame_Locked(scoped_refptr<VideoFrame> frame) {
775 DCHECK(task_runner_->RunsTasksInCurrentSequence());
776 lock_.AssertAcquired();
777 DCHECK(!frame->metadata().end_of_stream);
779 ++stats_.video_frames_decoded;
781 if (frame->metadata().power_efficient)
782 ++stats_.video_frames_decoded_power_efficient;
784 algorithm_->EnqueueFrame(std::move(frame));
787 void VideoRendererImpl::AttemptRead_Locked() {
788 DCHECK(task_runner_->RunsTasksInCurrentSequence());
789 lock_.AssertAcquired();
791 if (pending_read_ || received_end_of_stream_)
794 if (HaveReachedBufferingCap(max_buffered_frames_))
799 pending_read_ = true;
800 video_decoder_stream_->Read(
801 base::BindOnce(&VideoRendererImpl::FrameReady,
802 cancel_on_flush_weak_factory_.GetWeakPtr()));
812 void VideoRendererImpl::OnVideoDecoderStreamResetDone() {
813 // We don't need to acquire the |lock_| here, because we can only get here
814 // when Flush is in progress, so rendering and video sink must be stopped.
815 DCHECK(task_runner_->RunsTasksInCurrentSequence());
816 DCHECK(!sink_started_);
817 DCHECK_EQ(kFlushing, state_);
818 DCHECK(!received_end_of_stream_);
819 DCHECK(!rendered_end_of_stream_);
820 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
826 void VideoRendererImpl::UpdateStats_Locked(bool force_update) {
827 DCHECK(task_runner_->RunsTasksInCurrentSequence());
828 lock_.AssertAcquired();
830 // No need to check for `stats_.video_frames_decoded_power_efficient` because
831 // if it is greater than 0, `stats_.video_frames_decoded` will too.
832 if (!force_update && !stats_.video_frames_decoded &&
833 !stats_.video_frames_dropped) {
837 if (stats_.video_frames_dropped) {
838 TRACE_EVENT_INSTANT2("media", "VideoFramesDropped",
839 TRACE_EVENT_SCOPE_THREAD, "count",
840 stats_.video_frames_dropped, "id", player_id_);
843 const size_t memory_usage = algorithm_->GetMemoryUsage();
844 stats_.video_memory_usage = memory_usage - stats_.video_memory_usage;
845 stats_.video_frame_duration_average = algorithm_->average_frame_duration();
846 OnStatisticsUpdate(stats_);
848 stats_.video_frames_decoded = 0;
849 stats_.video_frames_dropped = 0;
850 stats_.video_frames_decoded_power_efficient = 0;
851 stats_.video_memory_usage = memory_usage;
854 void VideoRendererImpl::ReportFrameRateIfNeeded_Locked() {
855 DCHECK(task_runner_->RunsTasksInCurrentSequence());
856 lock_.AssertAcquired();
858 absl::optional<int> current_fps = fps_estimator_.ComputeFPS();
859 if (last_reported_fps_ && current_fps &&
860 *last_reported_fps_ == *current_fps) {
861 // Reported an FPS before, and it hasn't changed.
863 } else if (!last_reported_fps_ && !current_fps) {
864 // Did not report an FPS before, and we still don't have one
868 // FPS changed, possibly to unknown.
869 last_reported_fps_ = current_fps;
870 client_->OnVideoFrameRateChange(current_fps);
873 bool VideoRendererImpl::HaveReachedBufferingCap(size_t buffering_cap) const {
874 DCHECK(task_runner_->RunsTasksInCurrentSequence());
876 // When the display rate is less than the frame rate, the effective frames
877 // queued may be much smaller than the actual number of frames queued. Here
878 // we ensure that frames_queued() doesn't get excessive.
879 return algorithm_->effective_frames_queued() >= buffering_cap ||
880 algorithm_->frames_queued() >= kAbsoluteMaxFrames;
883 void VideoRendererImpl::StartSink() {
884 DCHECK(task_runner_->RunsTasksInCurrentSequence());
885 DCHECK_GT(algorithm_->frames_queued(), 0u);
886 sink_started_ = true;
887 was_background_rendering_ = false;
891 void VideoRendererImpl::StopSink() {
892 DCHECK(task_runner_->RunsTasksInCurrentSequence());
894 algorithm_->set_time_stopped();
895 sink_started_ = false;
896 was_background_rendering_ = false;
899 void VideoRendererImpl::MaybeFireEndedCallback_Locked(bool time_progressing) {
900 lock_.AssertAcquired();
902 // If there's only one frame in the video or Render() was never called, the
903 // algorithm will have one frame linger indefinitely. So in cases where the
904 // frame duration is unknown and we've received EOS, fire it once we get down
905 // to a single frame.
907 // Don't fire ended if we haven't received EOS or have already done so.
908 if (!received_end_of_stream_ || rendered_end_of_stream_)
911 const bool have_frames_after_start_time =
912 algorithm_->frames_queued() > 1 &&
913 !IsBeforeStartTime(algorithm_->last_frame());
915 // Don't fire ended if time isn't moving and we have frames.
916 if (!time_progressing && have_frames_after_start_time)
919 // Fire ended if we have no more effective frames, only ever had one frame, or
920 // we only have 1 effective frame and there's less than one render interval
921 // left before the ended event should execute.
922 base::TimeDelta ended_event_delay;
923 bool should_render_end_of_stream = false;
924 if (!algorithm_->effective_frames_queued()) {
925 // The best frame doesn't exist or was already rendered; end immediately.
926 should_render_end_of_stream = true;
927 } else if (algorithm_->frames_queued() == 1u &&
928 (algorithm_->average_frame_duration().is_zero() ||
929 algorithm_->render_interval().is_zero() || !time_progressing)) {
930 // We'll end up here if playback never started or there was only one frame.
931 should_render_end_of_stream = true;
932 } else if (algorithm_->frames_queued() == 1u &&
933 algorithm_->effective_frames_queued() == 1 && time_progressing) {
934 const auto end_delay =
935 std::max(base::TimeDelta(),
936 algorithm_->last_frame_end_time() - tick_clock_->NowTicks());
938 // We should only be here if time is progressing, so only fire the ended
939 // event now if we have less than one render interval before our next check.
940 if (end_delay < algorithm_->render_interval()) {
941 should_render_end_of_stream = true;
942 ended_event_delay = end_delay;
946 if (!should_render_end_of_stream)
949 rendered_end_of_stream_ = true;
950 task_runner_->PostDelayedTask(
952 base::BindOnce(&VideoRendererImpl::OnPlaybackEnded,
953 cancel_on_flush_weak_factory_.GetWeakPtr()),
957 base::TimeTicks VideoRendererImpl::ConvertMediaTimestamp(
958 base::TimeDelta media_time) {
959 std::vector<base::TimeDelta> media_times(1, media_time);
960 std::vector<base::TimeTicks> wall_clock_times;
961 if (!wall_clock_time_cb_.Run(media_times, &wall_clock_times))
962 return base::TimeTicks();
963 return wall_clock_times[0];
966 base::TimeTicks VideoRendererImpl::GetCurrentMediaTimeAsWallClockTime() {
967 std::vector<base::TimeTicks> current_time;
968 wall_clock_time_cb_.Run(std::vector<base::TimeDelta>(), ¤t_time);
969 return current_time[0];
972 bool VideoRendererImpl::IsBeforeStartTime(const VideoFrame& frame) {
973 // Prefer the actual frame duration over the average if available.
974 return frame.timestamp() + frame.metadata().frame_duration.value_or(
975 last_decoder_stream_avg_duration_) <
979 bool VideoRendererImpl::HasBestFirstFrame(const VideoFrame& frame) {
980 // We have the best first frame in the queue if our current frame has a
981 // timestamp after `start_timestamp_` or straddles `start_timestamp_`.
982 return frame.timestamp() >= start_timestamp_ ||
983 frame.timestamp() + frame.metadata().frame_duration.value_or(
984 last_decoder_stream_avg_duration_) >
988 void VideoRendererImpl::RemoveFramesForUnderflowOrBackgroundRendering() {
989 // Nothing to do if frame dropping is disabled for testing or we have nothing.
990 if (!drop_frames_ || !algorithm_->frames_queued())
993 // If we're paused for prerolling (current time is 0), don't expire any
994 // frames. It's possible that during preroll |have_nothing| is false while
995 // |was_background_rendering_| is true. We differentiate this from actual
996 // background rendering by checking if current time is 0.
997 const base::TimeTicks current_time = GetCurrentMediaTimeAsWallClockTime();
998 if (current_time.is_null())
1001 // Background rendering updates may not be ticking fast enough to remove
1002 // expired frames, so provide a boost here by ensuring we don't exit the
1003 // decoding cycle too early. Dropped frames are not counted in this case.
1004 if (was_background_rendering_) {
1005 algorithm_->RemoveExpiredFrames(tick_clock_->NowTicks());
1009 // If we've paused for underflow, and still have no effective frames, clear
1010 // the entire queue. Note: this may cause slight inaccuracies in the number
1011 // of dropped frames since the frame may have been rendered before.
1012 if (!sink_started_ && !algorithm_->effective_frames_queued()) {
1013 stats_.video_frames_dropped += algorithm_->frames_queued();
1015 VideoRendererAlgorithm::ResetFlag::kPreserveNextFrameEstimates);
1016 painted_first_frame_ = false;
1017 paint_first_frame_cb_.Cancel();
1019 // It's possible in the background rendering case for us to expire enough
1020 // frames that we need to transition from HAVE_ENOUGH => HAVE_NOTHING. Just
1021 // calling this function will check if we need to transition or not.
1022 if (buffering_state_ == BUFFERING_HAVE_ENOUGH)
1023 TransitionToHaveNothing_Locked();
1027 // Use the current media wall clock time plus the frame duration since
1028 // RemoveExpiredFrames() is expecting the end point of an interval (it will
1029 // subtract from the given value). It's important to always call this so
1030 // that frame statistics are updated correctly.
1031 if (buffering_state_ == BUFFERING_HAVE_NOTHING) {
1032 stats_.video_frames_dropped += algorithm_->RemoveExpiredFrames(
1033 current_time + algorithm_->average_frame_duration());
1037 // If we reach this point, the normal rendering process will take care of
1038 // removing any expired frames.
1041 void VideoRendererImpl::CheckForMetadataChanges(VideoPixelFormat pixel_format,
1042 const gfx::Size& natural_size) {
1043 DCHECK(task_runner_->RunsTasksInCurrentSequence());
1045 // Notify client of size and opacity changes if this is the first frame
1046 // or if those have changed from the last frame.
1047 if (!have_renderered_frames_ || last_frame_natural_size_ != natural_size) {
1048 last_frame_natural_size_ = natural_size;
1049 client_->OnVideoNaturalSizeChange(last_frame_natural_size_);
1052 const bool is_opaque = IsOpaque(pixel_format);
1053 if (!have_renderered_frames_ || last_frame_opaque_ != is_opaque) {
1054 last_frame_opaque_ = is_opaque;
1055 client_->OnVideoOpacityChange(last_frame_opaque_);
1058 have_renderered_frames_ = true;
1061 void VideoRendererImpl::AttemptReadAndCheckForMetadataChanges(
1062 VideoPixelFormat pixel_format,
1063 const gfx::Size& natural_size) {
1064 base::AutoLock auto_lock(lock_);
1065 CheckForMetadataChanges(pixel_format, natural_size);
1066 AttemptRead_Locked();
1069 void VideoRendererImpl::PaintFirstFrame() {
1070 DCHECK(task_runner_->RunsTasksInCurrentSequence());
1071 if (painted_first_frame_ || sink_started_) {
1075 DCHECK(algorithm_->frames_queued());
1078 algorithm_->Render(base::TimeTicks(), base::TimeTicks(), nullptr);
1079 DCHECK(first_frame);
1080 CheckForMetadataChanges(first_frame->format(), first_frame->natural_size());
1081 sink_->PaintSingleFrame(first_frame);
1082 painted_first_frame_ = true;
1083 paint_first_frame_cb_.Cancel();
1086 } // namespace media