media/renderers/video_renderer_impl.cc

   1 // Copyright 2013 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/renderers/video_renderer_impl.h"
   6
   7 #include <algorithm>
   8 #include <memory>
   9 #include <utility>
  10
  11 #include "base/feature_list.h"
  12 #include "base/functional/bind.h"
  13 #include "base/functional/callback.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/metrics/histogram_macros.h"
  17 #include "base/numerics/safe_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/task/bind_post_task.h"
  20 #include "base/task/sequenced_task_runner.h"
  21 #include "base/task/single_thread_task_runner.h"
  22 #include "base/time/default_tick_clock.h"
  23 #include "base/trace_event/trace_event.h"
  24 #include "media/base/media_log.h"
  25 #include "media/base/media_switches.h"
  26 #include "media/base/pipeline_status.h"
  27 #include "media/base/renderer_client.h"
  28 #include "media/base/video_frame.h"
  29
  30 namespace media {
  31
  32 namespace {
  33
  34 // Maximum number of frames we will buffer, regardless of their "effectiveness".
  35 // See HaveReachedBufferingCap(). The value was historically described in terms
  36 // of |min_buffered_frames_| as follows:
  37 // = 3 * high_water_mark(min_buffered_frames_),
  38 // = 3 * (2 * limits::kMaxVideoFrames)
  39 // = 3 * 2 * 4
  40 // Today, |min_buffered_frames_| can go down (as low as 1) and up in response to
  41 // SetLatencyHint(), so we needed to peg this with a constant.
  42 constexpr int kAbsoluteMaxFrames = 24;
  43
  44 bool ShouldUseLowDelayMode(DemuxerStream* stream) {
  45   return base::FeatureList::IsEnabled(kLowDelayVideoRenderingOnLiveStream) &&
  46          stream->liveness() == StreamLiveness::kLive;
  47 }
  48
  49 }  // namespace
  50
  51 VideoRendererImpl::VideoRendererImpl(
  52     const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
  53     VideoRendererSink* sink,
  54     const CreateVideoDecodersCB& create_video_decoders_cb,
  55     bool drop_frames,
  56     MediaLog* media_log,
  57     std::unique_ptr<GpuMemoryBufferVideoFramePool> gmb_pool,
  58     MediaPlayerLoggingID media_player_id)
  59     : task_runner_(media_task_runner),
  60       sink_(sink),
  61       sink_started_(false),
  62       client_(nullptr),
  63       gpu_memory_buffer_pool_(std::move(gmb_pool)),
  64       media_log_(media_log),
  65       player_id_(media_player_id),
  66       low_delay_(false),
  67       received_end_of_stream_(false),
  68       rendered_end_of_stream_(false),
  69       state_(kUninitialized),
  70       create_video_decoders_cb_(create_video_decoders_cb),
  71       pending_read_(false),
  72       drop_frames_(drop_frames),
  73       buffering_state_(BUFFERING_HAVE_NOTHING),
  74       tick_clock_(base::DefaultTickClock::GetInstance()),
  75       was_background_rendering_(false),
  76       time_progressing_(false),
  77       have_renderered_frames_(false),
  78       last_frame_opaque_(false),
  79       painted_first_frame_(false),
  80       min_buffered_frames_(initial_buffering_size_.value()),
  81       max_buffered_frames_(initial_buffering_size_.value()) {
  82   DCHECK(create_video_decoders_cb_);
  83 }
  84
  85 VideoRendererImpl::~VideoRendererImpl() {
  86   DCHECK(task_runner_->RunsTasksInCurrentSequence());
  87
  88   if (init_cb_)
  89     FinishInitialization(PIPELINE_ERROR_ABORT);
  90
  91   if (flush_cb_)
  92     FinishFlush();
  93
  94   if (sink_started_)
  95     StopSink();
  96 }
  97
  98 void VideoRendererImpl::Flush(base::OnceClosure callback) {
  99   DVLOG(1) << __func__;
 100   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 101
 102   if (sink_started_)
 103     StopSink();
 104
 105   base::AutoLock auto_lock(lock_);
 106
 107   DCHECK_EQ(state_, kPlaying);
 108   flush_cb_ = std::move(callback);
 109   state_ = kFlushing;
 110
 111   if (buffering_state_ != BUFFERING_HAVE_NOTHING) {
 112     buffering_state_ = BUFFERING_HAVE_NOTHING;
 113     task_runner_->PostTask(
 114         FROM_HERE,
 115         base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
 116                        weak_factory_.GetWeakPtr(), buffering_state_));
 117   }
 118   received_end_of_stream_ = false;
 119   rendered_end_of_stream_ = false;
 120
 121   // Reset |video_decoder_stream_| and drop any pending read callbacks from it.
 122   pending_read_ = false;
 123   if (gpu_memory_buffer_pool_)
 124     gpu_memory_buffer_pool_->Abort();
 125   cancel_on_flush_weak_factory_.InvalidateWeakPtrs();
 126   paint_first_frame_cb_.Cancel();
 127   video_decoder_stream_->Reset(
 128       base::BindOnce(&VideoRendererImpl::OnVideoDecoderStreamResetDone,
 129                      weak_factory_.GetWeakPtr()));
 130
 131   // To avoid unnecessary work by VDAs, only delete queued frames after
 132   // resetting |video_decoder_stream_|. If this is done in the opposite order
 133   // VDAs will get a bunch of ReusePictureBuffer() calls before the Reset(),
 134   // which they may use to output more frames that won't be used.
 135   algorithm_->Reset();
 136   painted_first_frame_ = false;
 137
 138   // Reset preroll capacity so seek time is not penalized. |latency_hint_|
 139   // and |low_delay_| mode disable automatic preroll adjustments.
 140   if (!latency_hint_.has_value() && !low_delay_) {
 141     min_buffered_frames_ = max_buffered_frames_ =
 142         initial_buffering_size_.value();
 143   }
 144 }
 145
 146 void VideoRendererImpl::StartPlayingFrom(base::TimeDelta timestamp) {
 147   DVLOG(1) << __func__ << "(" << timestamp.InMicroseconds() << ")";
 148   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 149   base::AutoLock auto_lock(lock_);
 150   DCHECK_EQ(state_, kFlushed);
 151   DCHECK(!pending_read_);
 152   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 153
 154   state_ = kPlaying;
 155   start_timestamp_ = timestamp;
 156   painted_first_frame_ = false;
 157   last_render_time_ = last_frame_ready_time_ = base::TimeTicks();
 158   video_decoder_stream_->SkipPrepareUntil(start_timestamp_);
 159   AttemptRead_Locked();
 160 }
 161
 162 void VideoRendererImpl::Initialize(
 163     DemuxerStream* stream,
 164     CdmContext* cdm_context,
 165     RendererClient* client,
 166     const TimeSource::WallClockTimeCB& wall_clock_time_cb,
 167     PipelineStatusCallback init_cb) {
 168   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 169   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "VideoRendererImpl::Initialize",
 170                                     TRACE_ID_LOCAL(this));
 171
 172   base::AutoLock auto_lock(lock_);
 173   DCHECK(stream);
 174   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
 175   DCHECK(init_cb);
 176   DCHECK(wall_clock_time_cb);
 177   DCHECK(kUninitialized == state_ || kFlushed == state_);
 178   DCHECK(!was_background_rendering_);
 179   DCHECK(!time_progressing_);
 180
 181   demuxer_stream_ = stream;
 182
 183   video_decoder_stream_ = std::make_unique<VideoDecoderStream>(
 184       std::make_unique<VideoDecoderStream::StreamTraits>(media_log_),
 185       task_runner_, create_video_decoders_cb_, media_log_);
 186   video_decoder_stream_->set_config_change_observer(base::BindRepeating(
 187       &VideoRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
 188   video_decoder_stream_->set_fallback_observer(base::BindRepeating(
 189       &VideoRendererImpl::OnFallback, weak_factory_.GetWeakPtr()));
 190   if (gpu_memory_buffer_pool_) {
 191     video_decoder_stream_->SetPrepareCB(base::BindRepeating(
 192         &GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame,
 193         // Safe since VideoDecoderStream won't issue calls after destruction.
 194         base::Unretained(gpu_memory_buffer_pool_.get())));
 195   }
 196
 197   low_delay_ = ShouldUseLowDelayMode(demuxer_stream_);
 198   if (low_delay_) {
 199     MEDIA_LOG(DEBUG, media_log_) << "Video rendering in low delay mode.";
 200
 201     // "Low delay mode" means only one frame must be buffered to transition to
 202     // BUFFERING_HAVE_ENOUGH.
 203     min_buffered_frames_ = 1;
 204   }
 205
 206   // Always post |init_cb_| because |this| could be destroyed if initialization
 207   // failed.
 208   init_cb_ = base::BindPostTaskToCurrentDefault(std::move(init_cb));
 209
 210   client_ = client;
 211   wall_clock_time_cb_ = wall_clock_time_cb;
 212   state_ = kInitializing;
 213
 214   current_decoder_config_ = demuxer_stream_->video_decoder_config();
 215   DCHECK(current_decoder_config_.IsValidConfig());
 216
 217   video_decoder_stream_->Initialize(
 218       demuxer_stream_,
 219       base::BindOnce(&VideoRendererImpl::OnVideoDecoderStreamInitialized,
 220                      weak_factory_.GetWeakPtr()),
 221       cdm_context,
 222       base::BindRepeating(&VideoRendererImpl::OnStatisticsUpdate,
 223                           weak_factory_.GetWeakPtr()),
 224       base::BindRepeating(&VideoRendererImpl::OnWaiting,
 225                           weak_factory_.GetWeakPtr()));
 226 }
 227
 228 scoped_refptr<VideoFrame> VideoRendererImpl::Render(
 229     base::TimeTicks deadline_min,
 230     base::TimeTicks deadline_max,
 231     RenderingMode rendering_mode) {
 232   TRACE_EVENT_BEGIN1("media", "VideoRendererImpl::Render", "id", player_id_);
 233   base::AutoLock auto_lock(lock_);
 234   DCHECK_EQ(state_, kPlaying);
 235   last_render_time_ = tick_clock_->NowTicks();
 236
 237   size_t frames_dropped = 0;
 238   scoped_refptr<VideoFrame> result =
 239       algorithm_->Render(deadline_min, deadline_max, &frames_dropped);
 240
 241   // Due to how the |algorithm_| holds frames, this should never be null if
 242   // we've had a proper startup sequence.
 243   DCHECK(result);
 244
 245   const bool background_rendering =
 246       rendering_mode == RenderingMode::kBackground;
 247
 248   // Declare HAVE_NOTHING if we reach a state where we can't progress playback
 249   // any further.  We don't want to do this if we've already done so, reached
 250   // end of stream, or have frames available.  We also don't want to do this in
 251   // background rendering mode, as the frames aren't visible anyways.
 252   MaybeFireEndedCallback_Locked(true);
 253   if (buffering_state_ == BUFFERING_HAVE_ENOUGH && !received_end_of_stream_ &&
 254       !algorithm_->effective_frames_queued() && !background_rendering &&
 255       !was_background_rendering_) {
 256     // Do not set |buffering_state_| here as the lock in FrameReady() may be
 257     // held already and it fire the state changes in the wrong order.
 258     DVLOG(3) << __func__ << " posted TransitionToHaveNothing.";
 259     task_runner_->PostTask(
 260         FROM_HERE, base::BindOnce(&VideoRendererImpl::TransitionToHaveNothing,
 261                                   weak_factory_.GetWeakPtr()));
 262   }
 263
 264   // We don't count dropped frames in the background to avoid skewing the count
 265   // and impacting JavaScript visible metrics used by web developers.
 266   //
 267   // Just after resuming from background rendering, we also don't count the
 268   // dropped frames since they are likely just dropped due to being too old.
 269   if (!background_rendering && !was_background_rendering_)
 270     stats_.video_frames_dropped += frames_dropped;
 271   was_background_rendering_ = background_rendering;
 272
 273   // Always post this task, it will acquire new frames if necessary and since it
 274   // happens on another thread, even if we don't have room in the queue now, by
 275   // the time it runs (may be delayed up to 50ms for complex decodes!) we might.
 276   task_runner_->PostTask(
 277       FROM_HERE,
 278       base::BindOnce(&VideoRendererImpl::AttemptReadAndCheckForMetadataChanges,
 279                      weak_factory_.GetWeakPtr(), result->format(),
 280                      result->natural_size()));
 281
 282   TRACE_EVENT_END1("media", "VideoRendererImpl::Render", "frame",
 283                    result->AsHumanReadableString());
 284   return result;
 285 }
 286
 287 void VideoRendererImpl::OnFrameDropped() {
 288   base::AutoLock auto_lock(lock_);
 289   algorithm_->OnLastFrameDropped();
 290 }
 291
 292 base::TimeDelta VideoRendererImpl::GetPreferredRenderInterval() {
 293   base::AutoLock auto_lock(lock_);
 294   return algorithm_->average_frame_duration();
 295 }
 296
 297 void VideoRendererImpl::OnVideoDecoderStreamInitialized(bool success) {
 298   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 299   base::AutoLock auto_lock(lock_);
 300   DCHECK_EQ(state_, kInitializing);
 301
 302   if (!success) {
 303     state_ = kUninitialized;
 304     FinishInitialization(DECODER_ERROR_NOT_SUPPORTED);
 305     return;
 306   }
 307
 308   // We're all good! Consider ourselves flushed because we have not read any
 309   // frames yet.
 310   state_ = kFlushed;
 311
 312   algorithm_ =
 313       std::make_unique<VideoRendererAlgorithm>(wall_clock_time_cb_, media_log_);
 314   if (!drop_frames_)
 315     algorithm_->disable_frame_dropping();
 316
 317   FinishInitialization(PIPELINE_OK);
 318 }
 319
 320 void VideoRendererImpl::FinishInitialization(PipelineStatus status) {
 321   DCHECK(init_cb_);
 322   TRACE_EVENT_NESTABLE_ASYNC_END1("media", "VideoRendererImpl::Initialize",
 323                                   TRACE_ID_LOCAL(this), "status",
 324                                   PipelineStatusToString(status));
 325   std::move(init_cb_).Run(status);
 326 }
 327
 328 void VideoRendererImpl::FinishFlush() {
 329   DCHECK(flush_cb_);
 330   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "VideoRendererImpl::Flush",
 331                                   TRACE_ID_LOCAL(this));
 332   std::move(flush_cb_).Run();
 333 }
 334
 335 void VideoRendererImpl::OnPlaybackError(PipelineStatus error) {
 336   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 337   client_->OnError(error);
 338 }
 339
 340 void VideoRendererImpl::OnPlaybackEnded() {
 341   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 342   {
 343     // Send one last stats update so things like memory usage are correct.
 344     base::AutoLock auto_lock(lock_);
 345     UpdateStats_Locked(true);
 346   }
 347
 348   client_->OnEnded();
 349 }
 350
 351 void VideoRendererImpl::OnStatisticsUpdate(const PipelineStatistics& stats) {
 352   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 353   client_->OnStatisticsUpdate(stats);
 354 }
 355
 356 void VideoRendererImpl::OnBufferingStateChange(BufferingState buffering_state) {
 357   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 358
 359   // "Underflow" is only possible when playing. This avoids noise like blaming
 360   // the decoder for an "underflow" that is really just a seek.
 361   BufferingStateChangeReason reason = BUFFERING_CHANGE_REASON_UNKNOWN;
 362   if (state_ == kPlaying && buffering_state == BUFFERING_HAVE_NOTHING) {
 363     reason = video_decoder_stream_->is_demuxer_read_pending()
 364                  ? DEMUXER_UNDERFLOW
 365                  : DECODER_UNDERFLOW;
 366   }
 367
 368   media_log_->AddEvent<MediaLogEvent::kBufferingStateChanged>(
 369       SerializableBufferingState<SerializableBufferingStateType::kVideo>{
 370           buffering_state, reason});
 371
 372   client_->OnBufferingStateChange(buffering_state, reason);
 373 }
 374
 375 void VideoRendererImpl::OnWaiting(WaitingReason reason) {
 376   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 377   client_->OnWaiting(reason);
 378 }
 379
 380 void VideoRendererImpl::OnConfigChange(const VideoDecoderConfig& config) {
 381   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 382   DCHECK(config.IsValidConfig());
 383
 384   // RendererClient only cares to know about config changes that differ from
 385   // previous configs.
 386   if (!current_decoder_config_.Matches(config)) {
 387     current_decoder_config_ = config;
 388     client_->OnVideoConfigChange(config);
 389   }
 390 }
 391
 392 void VideoRendererImpl::OnFallback(PipelineStatus status) {
 393   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 394   client_->OnFallback(std::move(status).AddHere());
 395 }
 396
 397 void VideoRendererImpl::SetTickClockForTesting(
 398     const base::TickClock* tick_clock) {
 399   tick_clock_ = tick_clock;
 400 }
 401
 402 void VideoRendererImpl::OnTimeProgressing() {
 403   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 404
 405   // WARNING: Do not attempt to use |lock_| here as StartSink() may cause a
 406   // reentrant call.
 407
 408   time_progressing_ = true;
 409
 410   if (sink_started_)
 411     return;
 412
 413   // If only an EOS frame came in after a seek, the renderer may not have
 414   // received the ended event yet though we've posted it.
 415   if (rendered_end_of_stream_)
 416     return;
 417
 418   // If we have no frames queued, there is a pending buffering state change in
 419   // flight and we should ignore the start attempt.
 420   if (!algorithm_->frames_queued()) {
 421     DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 422     return;
 423   }
 424
 425   StartSink();
 426 }
 427
 428 void VideoRendererImpl::OnTimeStopped() {
 429   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 430
 431   // WARNING: Do not attempt to use |lock_| here as StopSink() may cause a
 432   // reentrant call.
 433
 434   time_progressing_ = false;
 435
 436   if (!sink_started_)
 437     return;
 438
 439   StopSink();
 440
 441   // Make sure we expire everything we can if we can't read any more currently,
 442   // otherwise playback may hang indefinitely.  Note: There are no effective
 443   // frames queued at this point, otherwise FrameReady() would have canceled
 444   // the underflow state before reaching this point.
 445   if (buffering_state_ == BUFFERING_HAVE_NOTHING) {
 446     base::AutoLock al(lock_);
 447     RemoveFramesForUnderflowOrBackgroundRendering();
 448
 449     // If we've underflowed, increase the number of frames required to reach
 450     // BUFFERING_HAVE_ENOUGH upon resume; this will help prevent us from
 451     // repeatedly underflowing. Providing a |latency_hint_| or enabling
 452     // |low_delay_| mode disables automatic increases. In these cases the site
 453     // is expressing a desire to manually control/minimize the buffering
 454     // threshold for HAVE_ENOUGH.
 455     const size_t kMaxUnderflowGrowth = 2 * initial_buffering_size_.value();
 456     if (!latency_hint_.has_value() && !low_delay_) {
 457       DCHECK_EQ(min_buffered_frames_, max_buffered_frames_);
 458
 459       if (min_buffered_frames_ < kMaxUnderflowGrowth) {
 460         min_buffered_frames_++;
 461         DVLOG(2) << __func__ << " Underflow! Increased min_buffered_frames_: "
 462                  << min_buffered_frames_;
 463       }
 464     }
 465
 466     // Increase |max_buffered_frames_| irrespective of |latency_hint_| and
 467     // |low_delay_| mode. Unlike |min_buffered_frames_|, this does not affect
 468     // the buffering threshold for HAVE_ENOUGH. When max > min, the renderer can
 469     // buffer frames _beyond_ the HAVE_ENOUGH threshold (assuming decoder is
 470     // fast enough), which still helps reduce the likelihood of repeat
 471     // underflow.
 472     if (max_buffered_frames_ < kMaxUnderflowGrowth) {
 473       max_buffered_frames_++;
 474       DVLOG(2) << __func__ << " Underflow! Increased max_buffered_frames_: "
 475                << max_buffered_frames_;
 476     }
 477   }
 478 }
 479
 480 void VideoRendererImpl::SetLatencyHint(
 481     absl::optional<base::TimeDelta> latency_hint) {
 482   base::AutoLock auto_lock(lock_);
 483
 484   latency_hint_ = latency_hint;
 485
 486   // Permanently disable implicit |low_delay_| mode. Apps using latencyHint
 487   // are taking manual control of how buffering works. Unsetting the hint
 488   // will make rendering behave as if |low_delay_| were never set.
 489   low_delay_ = false;
 490
 491   if (!latency_hint_.has_value()) {
 492     // Restore default values.
 493     // NOTE |initial_buffering_size_| the default max, not the max overall.
 494     min_buffered_frames_ = max_buffered_frames_ =
 495         initial_buffering_size_.value();
 496     MEDIA_LOG(DEBUG, media_log_)
 497         << "Video latency hint cleared. Default buffer size ("
 498         << min_buffered_frames_ << " frames) restored";
 499   } else if (latency_hint_->is_zero()) {
 500     // Zero is a special case implying the bare minimum buffering (1 frame).
 501     // We apply the hint here outside of UpdateLatencyHintBufferingCaps_Locked()
 502     // to avoid needless churn since the "bare minimum" buffering doesn't
 503     // fluctuate with changes to FPS.
 504     min_buffered_frames_ = 1;
 505     max_buffered_frames_ = initial_buffering_size_.value();
 506     MEDIA_LOG(DEBUG, media_log_)
 507         << "Video latency hint set:" << *latency_hint << ". "
 508         << "Effective buffering latency: 1 frame";
 509   } else {
 510     // Non-zero latency hints are set here. Update buffering caps immediately if
 511     // we already have an algorithm_. Otherwise, the update will be applied as
 512     // frames arrive and duration becomes known. The caps will be recalculated
 513     // for each frame in case |average_frame_druation| changes.
 514     // |is_latency_hint_media_logged_| ensures that we only MEDIA_LOG on the
 515     // first application of this hint.
 516     is_latency_hint_media_logged_ = false;
 517     if (algorithm_) {
 518       UpdateLatencyHintBufferingCaps_Locked(
 519           algorithm_->average_frame_duration());
 520     }
 521   }
 522 }
 523
 524 void VideoRendererImpl::UpdateLatencyHintBufferingCaps_Locked(
 525     base::TimeDelta average_frame_duration) {
 526   lock_.AssertAcquired();
 527
 528   // NOTE: this method may be called for every frame. Only perform trivial
 529   // tasks.
 530
 531   // This method should only be called for non-zero latency hints. Zero is hard
 532   // coded to 1 frame inside SetLatencyHint().
 533   DCHECK(latency_hint_.has_value() && !latency_hint_->is_zero());
 534
 535   // For hints > 0, we need |average_frame_duration| to determine how many
 536   // frames would yield the specified target latency. This method will be called
 537   // again as |average_frame_duration| changes.
 538   if (average_frame_duration.is_zero())
 539     return;
 540
 541   int latency_hint_frames =
 542       base::ClampRound(*latency_hint_ / average_frame_duration);
 543
 544   std::string clamp_string;
 545   if (latency_hint_frames > kAbsoluteMaxFrames) {
 546     min_buffered_frames_ = kAbsoluteMaxFrames;
 547     clamp_string = " (clamped to max)";
 548   } else if (latency_hint_frames < 1) {
 549     min_buffered_frames_ = 1;
 550     clamp_string = " (clamped to min)";
 551   } else {
 552     min_buffered_frames_ = latency_hint_frames;
 553   }
 554
 555   // Use initial capacity limit if possible. Increase if needed.
 556   max_buffered_frames_ =
 557       std::max(min_buffered_frames_, initial_buffering_size_.value());
 558
 559   if (!is_latency_hint_media_logged_) {
 560     is_latency_hint_media_logged_ = true;
 561     MEDIA_LOG(DEBUG, media_log_)
 562         << "Video latency hint set:" << *latency_hint_ << ". "
 563         << "Effective buffering latency:"
 564         << (min_buffered_frames_ * average_frame_duration) << clamp_string;
 565   }
 566 }
 567
 568 void VideoRendererImpl::FrameReady(VideoDecoderStream::ReadResult result) {
 569   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 570   base::AutoLock auto_lock(lock_);
 571   DCHECK_EQ(state_, kPlaying);
 572   CHECK(pending_read_);
 573   pending_read_ = false;
 574
 575   // Can happen when demuxers are preparing for a new Seek().
 576   switch (result.code()) {
 577     case DecoderStatus::Codes::kOk:
 578       break;
 579     case DecoderStatus::Codes::kAborted:
 580       // TODO(liberato): This used to check specifically for the value
 581       // DEMUXER_READ_ABORTED, which was more specific than |kAborted|.
 582       // However, since it's a dcheck, this seems okay.
 583       return;
 584     default:
 585       // Anything other than `kOk` or `kAborted` is treated as an error.
 586       DCHECK(!result.has_value());
 587
 588       PipelineStatus::Codes code =
 589           result.code() == DecoderStatus::Codes::kDisconnected
 590               ? PIPELINE_ERROR_DISCONNECTED
 591               : PIPELINE_ERROR_DECODE;
 592       PipelineStatus status = {code, std::move(result).error()};
 593       task_runner_->PostTask(
 594           FROM_HERE,
 595           base::BindOnce(&VideoRendererImpl::OnPlaybackError,
 596                          weak_factory_.GetWeakPtr(), std::move(status)));
 597       return;
 598   }
 599
 600   DCHECK(result.has_value());
 601   scoped_refptr<VideoFrame> frame = std::move(result).value();
 602   DCHECK(frame);
 603
 604   last_frame_ready_time_ = tick_clock_->NowTicks();
 605   last_decoder_stream_avg_duration_ = video_decoder_stream_->AverageDuration();
 606
 607   const bool is_eos = frame->metadata().end_of_stream;
 608   const bool is_before_start_time = !is_eos && IsBeforeStartTime(*frame);
 609   const bool cant_read = !video_decoder_stream_->CanReadWithoutStalling();
 610   const bool has_best_first_frame = !is_eos && HasBestFirstFrame(*frame);
 611   const auto format = frame->format();
 612   const auto natural_size = frame->natural_size();
 613
 614   if (is_eos) {
 615     DCHECK(!received_end_of_stream_);
 616     received_end_of_stream_ = true;
 617     fps_estimator_.Reset();
 618     ReportFrameRateIfNeeded_Locked();
 619   } else if ((min_buffered_frames_ == 1 || cant_read) && is_before_start_time) {
 620     // Don't accumulate frames that are earlier than the start time if we
 621     // won't have a chance for a better frame, otherwise we could declare
 622     // HAVE_ENOUGH_DATA and start playback prematurely.
 623     fps_estimator_.Reset();
 624     ReportFrameRateIfNeeded_Locked();
 625     AttemptRead_Locked();
 626     return;
 627   } else {
 628     // If the sink hasn't been started, we still have time to release less
 629     // than ideal frames prior to startup.  We don't use IsBeforeStartTime()
 630     // here since it's based on a duration estimate and we can be exact here.
 631     if (!sink_started_ && frame->timestamp() <= start_timestamp_) {
 632       algorithm_->Reset();
 633       fps_estimator_.Reset();
 634       ReportFrameRateIfNeeded_Locked();
 635     }
 636
 637     // Provide frame duration information so that even if we only have one frame
 638     // in the queue we can properly estimate duration. This allows the call to
 639     // RemoveFramesForUnderflowOrBackgroundRendering() below to actually expire
 640     // this frame if it's too far behind the current media time. Without this,
 641     // we may resume too soon after a track change in the low delay case.
 642     if (!frame->metadata().frame_duration.has_value())
 643       frame->metadata().frame_duration = last_decoder_stream_avg_duration_;
 644
 645     AddReadyFrame_Locked(std::move(frame));
 646   }
 647
 648   // Attempt to purge bad frames in case of underflow or backgrounding.
 649   RemoveFramesForUnderflowOrBackgroundRendering();
 650
 651   // Paint the first frame if possible and necessary. Paint ahead of
 652   // HAVE_ENOUGH_DATA to ensure the user sees the frame as early as possible.
 653   // Paint before calling algorithm_->average_frame_duration(), as the call to
 654   // Render() will trigger internal duration updates.
 655   //
 656   // We want to paint the first frame under two conditions: Either (1) we have
 657   // enough frames to know it's definitely the first frame or (2) there may be
 658   // no more frames coming (sometimes unless we paint one of them).
 659   //
 660   // We have to check both effective_frames_queued() and |has_best_first_frame|
 661   // since prior to the clock starting effective_frames_queued() is a guess.
 662   //
 663   // NOTE: Do this before using algorithm_->average_frame_duration(). This
 664   // initial render will update the duration to be non-zero when provided by
 665   // frame metadata.
 666   if (!sink_started_ && !painted_first_frame_ && algorithm_->frames_queued()) {
 667     if (received_end_of_stream_ ||
 668         (algorithm_->effective_frames_queued() && has_best_first_frame)) {
 669       PaintFirstFrame();
 670     } else if (cant_read) {
 671       // `cant_read` isn't always reliable, so only paint after 250ms if we
 672       // haven't gotten anything better. This resets for each frame received. We
 673       // still kick off any metadata changes to avoid any layout shift though.
 674       CheckForMetadataChanges(format, natural_size);
 675       paint_first_frame_cb_.Reset(base::BindOnce(
 676           &VideoRendererImpl::PaintFirstFrame, base::Unretained(this)));
 677       task_runner_->PostDelayedTask(FROM_HERE, paint_first_frame_cb_.callback(),
 678                                     base::Milliseconds(250));
 679     }
 680   }
 681
 682   // Update average frame duration.
 683   base::TimeDelta frame_duration = algorithm_->average_frame_duration();
 684   if (frame_duration != kNoTimestamp && frame_duration != base::Seconds(0)) {
 685     fps_estimator_.AddSample(frame_duration);
 686   } else {
 687     fps_estimator_.Reset();
 688   }
 689   ReportFrameRateIfNeeded_Locked();
 690
 691   // Update any statistics since the last call.
 692   UpdateStats_Locked();
 693
 694   // Update hint-driven buffering caps to use the latest average frame duration.
 695   // NOTE: Do this before updating the buffering state below, as it may affect
 696   // the outcome of HaveEnoughData_Locked().
 697   // TODO(chcunningham): Duration from |algorithm_| is affected by playback
 698   // rate. Consider using wall clock frame duration instead.
 699   if (latency_hint_.has_value() && !latency_hint_->is_zero())
 700     UpdateLatencyHintBufferingCaps_Locked(frame_duration);
 701
 702   // Signal buffering state if we've met our conditions.
 703   if (buffering_state_ == BUFFERING_HAVE_NOTHING && HaveEnoughData_Locked())
 704     TransitionToHaveEnough_Locked();
 705
 706   // We may have removed all frames above and have reached end of stream. This
 707   // must happen after the buffering state change has been signaled.
 708   MaybeFireEndedCallback_Locked(time_progressing_);
 709
 710   // Always request more decoded video if we have capacity.
 711   AttemptRead_Locked();
 712 }
 713
 714 bool VideoRendererImpl::HaveEnoughData_Locked() const {
 715   DCHECK_EQ(state_, kPlaying);
 716   lock_.AssertAcquired();
 717
 718   if (received_end_of_stream_)
 719     return true;
 720
 721   if (HaveReachedBufferingCap(min_buffered_frames_))
 722     return true;
 723
 724   // If we've decoded any frames since the last render, signal have enough to
 725   // avoid underflowing when video is not visible unless we run out of frames.
 726   if (was_background_rendering_ && last_frame_ready_time_ >= last_render_time_)
 727     return true;
 728
 729   if (min_buffered_frames_ > 1 &&
 730       video_decoder_stream_->CanReadWithoutStalling()) {
 731     return false;
 732   }
 733
 734   // Note: We still require an effective frame in the stalling case since this
 735   // method is also used to inform TransitionToHaveNothing_Locked() and thus
 736   // would never pause and rebuffer if we always return true here.
 737   return algorithm_->effective_frames_queued() > 0u;
 738 }
 739
 740 void VideoRendererImpl::TransitionToHaveEnough_Locked() {
 741   DVLOG(3) << __func__;
 742   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 743   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 744   lock_.AssertAcquired();
 745
 746   buffering_state_ = BUFFERING_HAVE_ENOUGH;
 747   task_runner_->PostTask(
 748       FROM_HERE, base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
 749                                 weak_factory_.GetWeakPtr(), buffering_state_));
 750 }
 751
 752 void VideoRendererImpl::TransitionToHaveNothing() {
 753   DVLOG(3) << __func__;
 754   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 755
 756   base::AutoLock auto_lock(lock_);
 757   TransitionToHaveNothing_Locked();
 758 }
 759
 760 void VideoRendererImpl::TransitionToHaveNothing_Locked() {
 761   DVLOG(3) << __func__;
 762   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 763   lock_.AssertAcquired();
 764
 765   if (buffering_state_ != BUFFERING_HAVE_ENOUGH || HaveEnoughData_Locked())
 766     return;
 767
 768   buffering_state_ = BUFFERING_HAVE_NOTHING;
 769   task_runner_->PostTask(
 770       FROM_HERE, base::BindOnce(&VideoRendererImpl::OnBufferingStateChange,
 771                                 weak_factory_.GetWeakPtr(), buffering_state_));
 772 }
 773
 774 void VideoRendererImpl::AddReadyFrame_Locked(scoped_refptr<VideoFrame> frame) {
 775   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 776   lock_.AssertAcquired();
 777   DCHECK(!frame->metadata().end_of_stream);
 778
 779   ++stats_.video_frames_decoded;
 780
 781   if (frame->metadata().power_efficient)
 782     ++stats_.video_frames_decoded_power_efficient;
 783
 784   algorithm_->EnqueueFrame(std::move(frame));
 785 }
 786
 787 void VideoRendererImpl::AttemptRead_Locked() {
 788   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 789   lock_.AssertAcquired();
 790
 791   if (pending_read_ || received_end_of_stream_)
 792     return;
 793
 794   if (HaveReachedBufferingCap(max_buffered_frames_))
 795     return;
 796
 797   switch (state_) {
 798     case kPlaying:
 799       pending_read_ = true;
 800       video_decoder_stream_->Read(
 801           base::BindOnce(&VideoRendererImpl::FrameReady,
 802                          cancel_on_flush_weak_factory_.GetWeakPtr()));
 803       return;
 804     case kUninitialized:
 805     case kInitializing:
 806     case kFlushing:
 807     case kFlushed:
 808       return;
 809   }
 810 }
 811
 812 void VideoRendererImpl::OnVideoDecoderStreamResetDone() {
 813   // We don't need to acquire the |lock_| here, because we can only get here
 814   // when Flush is in progress, so rendering and video sink must be stopped.
 815   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 816   DCHECK(!sink_started_);
 817   DCHECK_EQ(kFlushing, state_);
 818   DCHECK(!received_end_of_stream_);
 819   DCHECK(!rendered_end_of_stream_);
 820   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 821
 822   state_ = kFlushed;
 823   FinishFlush();
 824 }
 825
 826 void VideoRendererImpl::UpdateStats_Locked(bool force_update) {
 827   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 828   lock_.AssertAcquired();
 829
 830   // No need to check for `stats_.video_frames_decoded_power_efficient` because
 831   // if it is greater than 0, `stats_.video_frames_decoded` will too.
 832   if (!force_update && !stats_.video_frames_decoded &&
 833       !stats_.video_frames_dropped) {
 834     return;
 835   }
 836
 837   if (stats_.video_frames_dropped) {
 838     TRACE_EVENT_INSTANT2("media", "VideoFramesDropped",
 839                          TRACE_EVENT_SCOPE_THREAD, "count",
 840                          stats_.video_frames_dropped, "id", player_id_);
 841   }
 842
 843   const size_t memory_usage = algorithm_->GetMemoryUsage();
 844   stats_.video_memory_usage = memory_usage - stats_.video_memory_usage;
 845   stats_.video_frame_duration_average = algorithm_->average_frame_duration();
 846   OnStatisticsUpdate(stats_);
 847
 848   stats_.video_frames_decoded = 0;
 849   stats_.video_frames_dropped = 0;
 850   stats_.video_frames_decoded_power_efficient = 0;
 851   stats_.video_memory_usage = memory_usage;
 852 }
 853
 854 void VideoRendererImpl::ReportFrameRateIfNeeded_Locked() {
 855   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 856   lock_.AssertAcquired();
 857
 858   absl::optional<int> current_fps = fps_estimator_.ComputeFPS();
 859   if (last_reported_fps_ && current_fps &&
 860       *last_reported_fps_ == *current_fps) {
 861     // Reported an FPS before, and it hasn't changed.
 862     return;
 863   } else if (!last_reported_fps_ && !current_fps) {
 864     // Did not report an FPS before, and we still don't have one
 865     return;
 866   }
 867
 868   // FPS changed, possibly to unknown.
 869   last_reported_fps_ = current_fps;
 870   client_->OnVideoFrameRateChange(current_fps);
 871 }
 872
 873 bool VideoRendererImpl::HaveReachedBufferingCap(size_t buffering_cap) const {
 874   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 875
 876   // When the display rate is less than the frame rate, the effective frames
 877   // queued may be much smaller than the actual number of frames queued.  Here
 878   // we ensure that frames_queued() doesn't get excessive.
 879   return algorithm_->effective_frames_queued() >= buffering_cap ||
 880          algorithm_->frames_queued() >= kAbsoluteMaxFrames;
 881 }
 882
 883 void VideoRendererImpl::StartSink() {
 884   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 885   DCHECK_GT(algorithm_->frames_queued(), 0u);
 886   sink_started_ = true;
 887   was_background_rendering_ = false;
 888   sink_->Start(this);
 889 }
 890
 891 void VideoRendererImpl::StopSink() {
 892   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 893   sink_->Stop();
 894   algorithm_->set_time_stopped();
 895   sink_started_ = false;
 896   was_background_rendering_ = false;
 897 }
 898
 899 void VideoRendererImpl::MaybeFireEndedCallback_Locked(bool time_progressing) {
 900   lock_.AssertAcquired();
 901
 902   // If there's only one frame in the video or Render() was never called, the
 903   // algorithm will have one frame linger indefinitely.  So in cases where the
 904   // frame duration is unknown and we've received EOS, fire it once we get down
 905   // to a single frame.
 906
 907   // Don't fire ended if we haven't received EOS or have already done so.
 908   if (!received_end_of_stream_ || rendered_end_of_stream_)
 909     return;
 910
 911   const bool have_frames_after_start_time =
 912       algorithm_->frames_queued() > 1 &&
 913       !IsBeforeStartTime(algorithm_->last_frame());
 914
 915   // Don't fire ended if time isn't moving and we have frames.
 916   if (!time_progressing && have_frames_after_start_time)
 917     return;
 918
 919   // Fire ended if we have no more effective frames, only ever had one frame, or
 920   // we only have 1 effective frame and there's less than one render interval
 921   // left before the ended event should execute.
 922   base::TimeDelta ended_event_delay;
 923   bool should_render_end_of_stream = false;
 924   if (!algorithm_->effective_frames_queued()) {
 925     // The best frame doesn't exist or was already rendered; end immediately.
 926     should_render_end_of_stream = true;
 927   } else if (algorithm_->frames_queued() == 1u &&
 928              (algorithm_->average_frame_duration().is_zero() ||
 929               algorithm_->render_interval().is_zero() || !time_progressing)) {
 930     // We'll end up here if playback never started or there was only one frame.
 931     should_render_end_of_stream = true;
 932   } else if (algorithm_->frames_queued() == 1u &&
 933              algorithm_->effective_frames_queued() == 1 && time_progressing) {
 934     const auto end_delay =
 935         std::max(base::TimeDelta(),
 936                  algorithm_->last_frame_end_time() - tick_clock_->NowTicks());
 937
 938     // We should only be here if time is progressing, so only fire the ended
 939     // event now if we have less than one render interval before our next check.
 940     if (end_delay < algorithm_->render_interval()) {
 941       should_render_end_of_stream = true;
 942       ended_event_delay = end_delay;
 943     }
 944   }
 945
 946   if (!should_render_end_of_stream)
 947     return;
 948
 949   rendered_end_of_stream_ = true;
 950   task_runner_->PostDelayedTask(
 951       FROM_HERE,
 952       base::BindOnce(&VideoRendererImpl::OnPlaybackEnded,
 953                      cancel_on_flush_weak_factory_.GetWeakPtr()),
 954       ended_event_delay);
 955 }
 956
 957 base::TimeTicks VideoRendererImpl::ConvertMediaTimestamp(
 958     base::TimeDelta media_time) {
 959   std::vector<base::TimeDelta> media_times(1, media_time);
 960   std::vector<base::TimeTicks> wall_clock_times;
 961   if (!wall_clock_time_cb_.Run(media_times, &wall_clock_times))
 962     return base::TimeTicks();
 963   return wall_clock_times[0];
 964 }
 965
 966 base::TimeTicks VideoRendererImpl::GetCurrentMediaTimeAsWallClockTime() {
 967   std::vector<base::TimeTicks> current_time;
 968   wall_clock_time_cb_.Run(std::vector<base::TimeDelta>(), &current_time);
 969   return current_time[0];
 970 }
 971
 972 bool VideoRendererImpl::IsBeforeStartTime(const VideoFrame& frame) {
 973   // Prefer the actual frame duration over the average if available.
 974   return frame.timestamp() + frame.metadata().frame_duration.value_or(
 975                                  last_decoder_stream_avg_duration_) <
 976          start_timestamp_;
 977 }
 978
 979 bool VideoRendererImpl::HasBestFirstFrame(const VideoFrame& frame) {
 980   // We have the best first frame in the queue if our current frame has a
 981   // timestamp after `start_timestamp_` or straddles `start_timestamp_`.
 982   return frame.timestamp() >= start_timestamp_ ||
 983          frame.timestamp() + frame.metadata().frame_duration.value_or(
 984                                  last_decoder_stream_avg_duration_) >
 985              start_timestamp_;
 986 }
 987
 988 void VideoRendererImpl::RemoveFramesForUnderflowOrBackgroundRendering() {
 989   // Nothing to do if frame dropping is disabled for testing or we have nothing.
 990   if (!drop_frames_ || !algorithm_->frames_queued())
 991     return;
 992
 993   // If we're paused for prerolling (current time is 0), don't expire any
 994   // frames. It's possible that during preroll |have_nothing| is false while
 995   // |was_background_rendering_| is true. We differentiate this from actual
 996   // background rendering by checking if current time is 0.
 997   const base::TimeTicks current_time = GetCurrentMediaTimeAsWallClockTime();
 998   if (current_time.is_null())
 999     return;
1000
1001   // Background rendering updates may not be ticking fast enough to remove
1002   // expired frames, so provide a boost here by ensuring we don't exit the
1003   // decoding cycle too early. Dropped frames are not counted in this case.
1004   if (was_background_rendering_) {
1005     algorithm_->RemoveExpiredFrames(tick_clock_->NowTicks());
1006     return;
1007   }
1008
1009   // If we've paused for underflow, and still have no effective frames, clear
1010   // the entire queue.  Note: this may cause slight inaccuracies in the number
1011   // of dropped frames since the frame may have been rendered before.
1012   if (!sink_started_ && !algorithm_->effective_frames_queued()) {
1013     stats_.video_frames_dropped += algorithm_->frames_queued();
1014     algorithm_->Reset(
1015         VideoRendererAlgorithm::ResetFlag::kPreserveNextFrameEstimates);
1016     painted_first_frame_ = false;
1017     paint_first_frame_cb_.Cancel();
1018
1019     // It's possible in the background rendering case for us to expire enough
1020     // frames that we need to transition from HAVE_ENOUGH => HAVE_NOTHING. Just
1021     // calling this function will check if we need to transition or not.
1022     if (buffering_state_ == BUFFERING_HAVE_ENOUGH)
1023       TransitionToHaveNothing_Locked();
1024     return;
1025   }
1026
1027   // Use the current media wall clock time plus the frame duration since
1028   // RemoveExpiredFrames() is expecting the end point of an interval (it will
1029   // subtract from the given value). It's important to always call this so
1030   // that frame statistics are updated correctly.
1031   if (buffering_state_ == BUFFERING_HAVE_NOTHING) {
1032     stats_.video_frames_dropped += algorithm_->RemoveExpiredFrames(
1033         current_time + algorithm_->average_frame_duration());
1034     return;
1035   }
1036
1037   // If we reach this point, the normal rendering process will take care of
1038   // removing any expired frames.
1039 }
1040
1041 void VideoRendererImpl::CheckForMetadataChanges(VideoPixelFormat pixel_format,
1042                                                 const gfx::Size& natural_size) {
1043   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1044
1045   // Notify client of size and opacity changes if this is the first frame
1046   // or if those have changed from the last frame.
1047   if (!have_renderered_frames_ || last_frame_natural_size_ != natural_size) {
1048     last_frame_natural_size_ = natural_size;
1049     client_->OnVideoNaturalSizeChange(last_frame_natural_size_);
1050   }
1051
1052   const bool is_opaque = IsOpaque(pixel_format);
1053   if (!have_renderered_frames_ || last_frame_opaque_ != is_opaque) {
1054     last_frame_opaque_ = is_opaque;
1055     client_->OnVideoOpacityChange(last_frame_opaque_);
1056   }
1057
1058   have_renderered_frames_ = true;
1059 }
1060
1061 void VideoRendererImpl::AttemptReadAndCheckForMetadataChanges(
1062     VideoPixelFormat pixel_format,
1063     const gfx::Size& natural_size) {
1064   base::AutoLock auto_lock(lock_);
1065   CheckForMetadataChanges(pixel_format, natural_size);
1066   AttemptRead_Locked();
1067 }
1068
1069 void VideoRendererImpl::PaintFirstFrame() {
1070   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1071   if (painted_first_frame_ || sink_started_) {
1072     return;
1073   }
1074
1075   DCHECK(algorithm_->frames_queued());
1076
1077   auto first_frame =
1078       algorithm_->Render(base::TimeTicks(), base::TimeTicks(), nullptr);
1079   DCHECK(first_frame);
1080   CheckForMetadataChanges(first_frame->format(), first_frame->natural_size());
1081   sink_->PaintSingleFrame(first_frame);
1082   painted_first_frame_ = true;
1083   paint_first_frame_cb_.Cancel();
1084 }
1085
1086 }  // namespace media