third_party/blink/renderer/modules/mediastream/media_stream_video_track.cc

   1 // Copyright 2013 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
   6
   7 #include <string>
   8 #include <utility>
   9
  10 #include "base/containers/contains.h"
  11 #include "base/functional/bind.h"
  12 #include "base/location.h"
  13 #include "base/metrics/histogram_functions.h"
  14 #include "base/metrics/histogram_macros.h"
  15 #include "base/ranges/algorithm.h"
  16 #include "base/task/bind_post_task.h"
  17 #include "base/task/sequenced_task_runner.h"
  18 #include "base/task/single_thread_task_runner.h"
  19 #include "base/time/time.h"
  20 #include "build/build_config.h"
  21 #include "media/base/limits.h"
  22 #include "media/capture/video_capture_types.h"
  23 #include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h"
  24 #include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h"
  25 #include "third_party/blink/public/web/web_local_frame.h"
  26 #include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_device.h"
  27 #include "third_party/blink/renderer/platform/allow_discouraged_type.h"
  28 #include "third_party/blink/renderer/platform/mediastream/media_stream_component_impl.h"
  29 #include "third_party/blink/renderer/platform/scheduler/public/main_thread.h"
  30 #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
  31 #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
  32 #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_std.h"
  33 #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
  34 #include "third_party/blink/renderer/platform/wtf/vector.h"
  35
  36 namespace blink {
  37 namespace {
  38
  39 // A lower-bound for the refresh interval.
  40 constexpr base::TimeDelta kLowerBoundRefreshInterval =
  41     base::Hertz(media::limits::kMaxFramesPerSecond);
  42
  43 // This alias mimics the definition of VideoCaptureDeliverFrameCB.
  44 using VideoCaptureDeliverFrameInternalCallback = WTF::CrossThreadFunction<void(
  45     scoped_refptr<media::VideoFrame> video_frame,
  46     base::TimeTicks estimated_capture_time)>;
  47
  48 // This alias mimics the definition of VideoCaptureNotifyFrameDroppedCB.
  49 using VideoCaptureNotifyFrameDroppedInternalCallback =
  50     WTF::CrossThreadFunction<void(media::VideoCaptureFrameDropReason)>;
  51
  52 // Mimics blink::EncodedVideoFrameCB
  53 using EncodedVideoFrameInternalCallback =
  54     WTF::CrossThreadFunction<void(scoped_refptr<EncodedVideoFrame> frame,
  55                                   base::TimeTicks estimated_capture_time)>;
  56
  57 base::TimeDelta ComputeRefreshIntervalFromBounds(
  58     const base::TimeDelta required_min_refresh_interval,
  59     const absl::optional<double>& min_frame_rate,
  60     const absl::optional<double>& max_frame_rate) {
  61   // Start with the default required refresh interval, and refine based on
  62   // constraints. If a minimum frameRate is provided, use that. Otherwise, use
  63   // the maximum frameRate if it happens to be less than the default.
  64   base::TimeDelta refresh_interval = required_min_refresh_interval;
  65   if (min_frame_rate.has_value())
  66     refresh_interval = base::Hertz(*min_frame_rate);
  67
  68   if (max_frame_rate.has_value()) {
  69     refresh_interval = std::max(refresh_interval, base::Hertz(*max_frame_rate));
  70   }
  71
  72   if (refresh_interval < kLowerBoundRefreshInterval)
  73     refresh_interval = kLowerBoundRefreshInterval;
  74
  75   return refresh_interval;
  76 }
  77
  78 void LogVideoFrameDropUMA(media::VideoCaptureFrameDropReason reason,
  79                           mojom::blink::MediaStreamType stream_type) {
  80   const int kEnumCount =
  81       static_cast<int>(media::VideoCaptureFrameDropReason::kMaxValue) + 1;
  82   UMA_HISTOGRAM_ENUMERATION("Media.VideoCapture.Track.FrameDrop", reason,
  83                             kEnumCount);
  84   switch (stream_type) {
  85     case mojom::blink::MediaStreamType::DEVICE_VIDEO_CAPTURE:
  86       UMA_HISTOGRAM_ENUMERATION(
  87           "Media.VideoCapture.Track.FrameDrop.DeviceCapture", reason,
  88           kEnumCount);
  89       break;
  90     case mojom::blink::MediaStreamType::GUM_TAB_VIDEO_CAPTURE:
  91       UMA_HISTOGRAM_ENUMERATION(
  92           "Media.VideoCapture.Track.FrameDrop.GumTabCapture", reason,
  93           kEnumCount);
  94       break;
  95     case mojom::blink::MediaStreamType::GUM_DESKTOP_VIDEO_CAPTURE:
  96       UMA_HISTOGRAM_ENUMERATION(
  97           "Media.VideoCapture.Track.FrameDrop.GumDesktopCapture", reason,
  98           kEnumCount);
  99       break;
 100     case mojom::blink::MediaStreamType::DISPLAY_VIDEO_CAPTURE:
 101       UMA_HISTOGRAM_ENUMERATION(
 102           "Media.VideoCapture.Track.FrameDrop.DisplayCapture", reason,
 103           kEnumCount);
 104       break;
 105     case mojom::blink::MediaStreamType::DISPLAY_VIDEO_CAPTURE_THIS_TAB:
 106       UMA_HISTOGRAM_ENUMERATION(
 107           "Media.VideoCapture.Track.FrameDrop.DisplayCaptureCurrentTab", reason,
 108           kEnumCount);
 109       break;
 110     case mojom::blink::MediaStreamType::DISPLAY_VIDEO_CAPTURE_SET:
 111       UMA_HISTOGRAM_ENUMERATION(
 112           "Media.VideoCapture.Track.FrameDrop.DisplayCaptureSet", reason,
 113           kEnumCount);
 114       break;
 115     case mojom::blink::MediaStreamType::NO_SERVICE:
 116     case mojom::blink::MediaStreamType::DEVICE_AUDIO_CAPTURE:
 117     case mojom::blink::MediaStreamType::GUM_TAB_AUDIO_CAPTURE:
 118     case mojom::blink::MediaStreamType::GUM_DESKTOP_AUDIO_CAPTURE:
 119     case mojom::blink::MediaStreamType::DISPLAY_AUDIO_CAPTURE:
 120     case mojom::blink::MediaStreamType::NUM_MEDIA_TYPES:
 121       break;
 122   }
 123 }
 124
 125 }  // namespace
 126
 127 // MediaStreamVideoTrack::FrameDeliverer is a helper class used for registering
 128 // VideoCaptureDeliverFrameCB/EncodedVideoFrameCB callbacks on the main render
 129 // thread to receive video frames on the video task runner. Frames are only
 130 // delivered to the sinks if the track is enabled. If the track is disabled, a
 131 // black frame is instead forwarded to the sinks at the same frame rate. A
 132 // disabled track does not forward data to encoded sinks.
 133 class MediaStreamVideoTrack::FrameDeliverer
 134     : public WTF::ThreadSafeRefCounted<FrameDeliverer> {
 135  public:
 136   using VideoSinkId = WebMediaStreamSink*;
 137
 138   FrameDeliverer(
 139       scoped_refptr<base::SingleThreadTaskRunner> main_render_task_runner,
 140       scoped_refptr<base::SequencedTaskRunner> video_task_runner,
 141       base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track,
 142       base::WeakPtr<MediaStreamVideoSource> media_stream_video_source,
 143       bool enabled,
 144       uint32_t sub_capture_target_version);
 145
 146   FrameDeliverer(const FrameDeliverer&) = delete;
 147   FrameDeliverer& operator=(const FrameDeliverer&) = delete;
 148
 149   // Sets whether the track is enabled or not. If getting enabled and encoded
 150   // output is enabled, the deliverer will wait until the next key frame before
 151   // it resumes producing encoded data.
 152   void SetEnabled(bool enabled, bool await_key_frame);
 153
 154   // Add |callback| to receive video frames on the video task runner.
 155   // Must be called on the main render thread.
 156   void AddCallback(VideoSinkId id, VideoCaptureDeliverFrameCB callback);
 157
 158   // Sets the frame dropped callback of the sink of frame |id].
 159   void SetNotifyFrameDroppedCallback(VideoSinkId id,
 160                                      VideoCaptureNotifyFrameDroppedCB callback);
 161
 162   // Add |callback| to receive encoded video frames on the video task runner.
 163   // Must be called on the main render thread.
 164   void AddEncodedCallback(VideoSinkId id, EncodedVideoFrameCB callback);
 165
 166   // Removes |callback| associated with |id| from receiving video frames if |id|
 167   // has been added. It is ok to call RemoveCallback even if the |id| has not
 168   // been added. Note that the added callback will be reset on the main thread.
 169   // Must be called on the main render thread.
 170   void RemoveCallback(VideoSinkId id);
 171
 172   // Removes encoded callback associated with |id| from receiving video frames
 173   // if |id| has been added. It is ok to call RemoveEncodedCallback even if the
 174   // |id| has not been added. Note that the added callback will be reset on the
 175   // main thread. Must be called on the main render thread.
 176   void RemoveEncodedCallback(VideoSinkId id);
 177
 178   // Triggers all registered callbacks with |frame| and |estimated_capture_time|
 179   // as parameters. Must be called on the video task runner.
 180   void DeliverFrameOnVideoTaskRunner(
 181       scoped_refptr<media::VideoFrame> frame,
 182       base::TimeTicks estimated_capture_time);
 183
 184   // A frame was dropped instead of delivered. This is the main handler of frame
 185   // drops: it updates dropped/discarded counters, invokes
 186   // LogFrameDroppedOnVideoTaskRunner() and notifies `callbacks_` (i.e. sinks)
 187   // that a frame was dropped.
 188   void OnFrameDroppedOnVideoTaskRunner(
 189       media::VideoCaptureFrameDropReason reason);
 190
 191   // Can be called from any task runner (is atomic).
 192   size_t deliverable_frames() const { return deliverable_frames_; }
 193   size_t discarded_frames() const { return discarded_frames_; }
 194   size_t dropped_frames() const { return dropped_frames_; }
 195
 196   // Triggers all encoded callbacks with |frame| and |estimated_capture_time|.
 197   // Must be called on the video task runner.
 198   void DeliverEncodedVideoFrameOnVideoTaskRunner(
 199       scoped_refptr<EncodedVideoFrame> frame,
 200       base::TimeTicks estimated_capture_time);
 201
 202   // Called when a sub-capture-target-version is acknowledged by the capture
 203   // module. After this, it is guaranteed that all subsequent frames will be
 204   // associated with a sub-capture-target-version that is >=
 205   // |sub_capture_target_version|. Must be called on the video task runner.
 206   void NewSubCaptureTargetVersionOnVideoTaskRunner(
 207       uint32_t sub_capture_target_version);
 208
 209   void SetIsRefreshingForMinFrameRate(bool is_refreshing_for_min_frame_rate);
 210
 211   void AddSubCaptureTargetVersionCallback(uint32_t sub_capture_target_version,
 212                                           base::OnceClosure callback);
 213   void RemoveSubCaptureTargetVersionCallback(
 214       uint32_t sub_capture_target_version);
 215
 216   // Performs logging and UMAs relating to frame drops. This includes both
 217   // frames dropped prior to delivery (OnFrameDroppedOnVideoTaskRunner) and
 218   // MediaStreamVideoTrack::OnSinkDroppedFrame().
 219   void LogFrameDroppedOnVideoTaskRunner(
 220       media::VideoCaptureFrameDropReason reason);
 221
 222   void SetEmitLogMessage(
 223       base::RepeatingCallback<void(const std::string&)> emit_log_message);
 224
 225  private:
 226   friend class WTF::ThreadSafeRefCounted<FrameDeliverer>;
 227
 228   // Struct containing sink id, frame delivery and frame dropped callbacks.
 229   struct VideoIdCallbacks {
 230     VideoSinkId id;
 231     VideoCaptureDeliverFrameInternalCallback deliver_frame;
 232     VideoCaptureNotifyFrameDroppedInternalCallback notify_frame_dropped;
 233   };
 234
 235   virtual ~FrameDeliverer();
 236   void AddCallbackOnVideoTaskRunner(
 237       VideoSinkId id,
 238       VideoCaptureDeliverFrameInternalCallback callback);
 239   void SetNotifyFrameDroppedCallbackOnVideoTaskRunner(
 240       VideoSinkId id,
 241       VideoCaptureNotifyFrameDroppedInternalCallback callback,
 242       const scoped_refptr<base::SingleThreadTaskRunner>& task_runner);
 243   void RemoveCallbackOnVideoTaskRunner(
 244       VideoSinkId id,
 245       const scoped_refptr<base::SingleThreadTaskRunner>& task_runner);
 246
 247   void AddEncodedCallbackOnVideoTaskRunner(
 248       VideoSinkId id,
 249       EncodedVideoFrameInternalCallback callback);
 250   void RemoveEncodedCallbackOnVideoTaskRunner(
 251       VideoSinkId id,
 252       const scoped_refptr<base::SingleThreadTaskRunner>& task_runner);
 253
 254   void SetEnabledOnVideoTaskRunner(bool enabled, bool await_key_frame);
 255
 256   void SetIsRefreshingForMinFrameRateOnVideoTaskRunner(
 257       bool is_refreshing_for_min_frame_rate);
 258
 259   void AddSubCaptureTargetVersionCallbackOnVideoTaskRunner(
 260       uint32_t sub_capture_target_version,
 261       WTF::CrossThreadOnceClosure callback);
 262   void RemoveSubCaptureTargetVersionCallbackOnVideoTaskRunner(
 263       uint32_t sub_capture_target_version);
 264
 265   // Returns a black frame where the size and time stamp is set to the same as
 266   // as in |reference_frame|.
 267   scoped_refptr<media::VideoFrame> GetBlackFrame(
 268       const media::VideoFrame& reference_frame);
 269
 270   // Used to DCHECK that AddCallback and RemoveCallback are called on the main
 271   // Render Thread.
 272   THREAD_CHECKER(main_render_thread_checker_);
 273   const scoped_refptr<base::SequencedTaskRunner> video_task_runner_;
 274   const scoped_refptr<base::SingleThreadTaskRunner> main_render_task_runner_;
 275
 276   base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track_;
 277   base::WeakPtr<MediaStreamVideoSource> media_stream_video_source_;
 278   const mojom::blink::MediaStreamType stream_type_;
 279
 280   bool enabled_;
 281   scoped_refptr<media::VideoFrame> black_frame_;
 282   bool emit_frame_drop_events_;
 283
 284   Vector<VideoIdCallbacks> callbacks_;
 285   HashMap<VideoSinkId, EncodedVideoFrameInternalCallback> encoded_callbacks_;
 286
 287   // Frame counters for the MediaStreamTrack Statistics API. The counters are
 288   // only incremented when the track is enabled (even though a disabled track
 289   // delivers black frames).
 290   std::atomic<size_t> deliverable_frames_ = 0;
 291   std::atomic<size_t> discarded_frames_ = 0;
 292   std::atomic<size_t> dropped_frames_ = 0;
 293
 294   // Helper methods for LogFrameDroppedOnVideoTaskRunner().
 295   void MaybeEmitFrameDropLogMessage(media::VideoCaptureFrameDropReason reason);
 296   void EmitLogMessage(const std::string& message);
 297   base::RepeatingCallback<void(const std::string&)> emit_log_message_;
 298   // States relating to frame drop logging and UMAs.
 299   struct FrameDropLogState {
 300     explicit FrameDropLogState(media::VideoCaptureFrameDropReason reason =
 301                                    media::VideoCaptureFrameDropReason::kNone);
 302
 303     int drop_count = 0;
 304     media::VideoCaptureFrameDropReason drop_reason =
 305         media::VideoCaptureFrameDropReason::kNone;
 306     bool max_log_count_exceeded = false;
 307   };
 308   FrameDropLogState frame_drop_log_state_;
 309   // Tracks how often each frame-drop reason was encountered to decide whether
 310   // or not to LOG the console.
 311   std::map<media::VideoCaptureFrameDropReason, int> frame_drop_log_counters_
 312       ALLOW_DISCOURAGED_TYPE("TODO(crbug.com/1481448)");
 313
 314   // Callbacks that will be invoked a single time when a
 315   // sub-capture-target-version is observed that is at least equal to the key.
 316   // The map itself (sub_capture_target_version_callbacks_) is bound to the
 317   // video task runner. The callbacks are bound to their respective threads
 318   // (BindPostTask).
 319   HashMap<uint32_t, WTF::CrossThreadOnceClosure>
 320       sub_capture_target_version_callbacks_;
 321
 322   bool await_next_key_frame_;
 323
 324   // This should only be accessed on the video task runner.
 325   bool is_refreshing_for_min_frame_rate_ = false;
 326
 327   // This monotonously increasing value indicates which
 328   // sub-capture-target-version is expected for delivered frames.
 329   uint32_t sub_capture_target_version_ = 0;
 330 };
 331
 332 MediaStreamVideoTrack::FrameDeliverer::FrameDropLogState::FrameDropLogState(
 333     media::VideoCaptureFrameDropReason reason)
 334     : drop_count((reason == media::VideoCaptureFrameDropReason::kNone) ? 0 : 1),
 335       drop_reason(reason),
 336       max_log_count_exceeded(false) {}
 337
 338 MediaStreamVideoTrack::FrameDeliverer::FrameDeliverer(
 339     scoped_refptr<base::SingleThreadTaskRunner> main_render_task_runner,
 340     scoped_refptr<base::SequencedTaskRunner> video_task_runner,
 341     base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track,
 342     base::WeakPtr<MediaStreamVideoSource> media_stream_video_source,
 343     bool enabled,
 344     uint32_t sub_capture_target_version)
 345     : video_task_runner_(std::move(video_task_runner)),
 346       main_render_task_runner_(main_render_task_runner),
 347       media_stream_video_track_(media_stream_video_track),
 348       media_stream_video_source_(media_stream_video_source),
 349       stream_type_(media_stream_video_source_->device().type),
 350       enabled_(enabled),
 351       emit_frame_drop_events_(true),
 352       await_next_key_frame_(false),
 353       sub_capture_target_version_(sub_capture_target_version) {
 354   DCHECK(video_task_runner_.get());
 355   DCHECK(main_render_task_runner_);
 356   SetEmitLogMessage(ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 357       &MediaStreamVideoTrack::FrameDeliverer::EmitLogMessage,
 358       WTF::CrossThreadUnretained(this))));
 359 }
 360
 361 MediaStreamVideoTrack::FrameDeliverer::~FrameDeliverer() {
 362   DCHECK(callbacks_.empty());
 363 }
 364
 365 void MediaStreamVideoTrack::FrameDeliverer::AddCallback(
 366     VideoSinkId id,
 367     VideoCaptureDeliverFrameCB callback) {
 368   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 369   PostCrossThreadTask(
 370       *video_task_runner_, FROM_HERE,
 371       CrossThreadBindOnce(&FrameDeliverer::AddCallbackOnVideoTaskRunner,
 372                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 373                           CrossThreadBindRepeating(std::move(callback))));
 374 }
 375
 376 void MediaStreamVideoTrack::FrameDeliverer::AddCallbackOnVideoTaskRunner(
 377     VideoSinkId id,
 378     VideoCaptureDeliverFrameInternalCallback callback) {
 379   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 380   callbacks_.push_back(VideoIdCallbacks{
 381       id, std::move(callback),
 382       CrossThreadBindRepeating([](media::VideoCaptureFrameDropReason) {})});
 383 }
 384
 385 void MediaStreamVideoTrack::FrameDeliverer::SetNotifyFrameDroppedCallback(
 386     VideoSinkId id,
 387     VideoCaptureNotifyFrameDroppedCB callback) {
 388   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 389   PostCrossThreadTask(
 390       *video_task_runner_, FROM_HERE,
 391       CrossThreadBindOnce(
 392           &FrameDeliverer::SetNotifyFrameDroppedCallbackOnVideoTaskRunner,
 393           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 394           CrossThreadBindRepeating(std::move(callback)),
 395           main_render_task_runner_));
 396 }
 397
 398 void MediaStreamVideoTrack::FrameDeliverer::
 399     SetNotifyFrameDroppedCallbackOnVideoTaskRunner(
 400         VideoSinkId id,
 401         VideoCaptureNotifyFrameDroppedInternalCallback callback,
 402         const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
 403   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 404   DVLOG(1) << __func__;
 405   for (auto& entry : callbacks_) {
 406     if (entry.id == id) {
 407       // Old callback destruction needs to happen on the specified task
 408       // runner.
 409       PostCrossThreadTask(
 410           *task_runner, FROM_HERE,
 411           CrossThreadBindOnce(
 412               [](VideoCaptureNotifyFrameDroppedInternalCallback) {},
 413               std::move(entry.notify_frame_dropped)));
 414       entry.notify_frame_dropped = std::move(callback);
 415     }
 416   }
 417 }
 418
 419 void MediaStreamVideoTrack::FrameDeliverer::AddEncodedCallback(
 420     VideoSinkId id,
 421     EncodedVideoFrameCB callback) {
 422   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 423   PostCrossThreadTask(
 424       *video_task_runner_, FROM_HERE,
 425       CrossThreadBindOnce(&FrameDeliverer::AddEncodedCallbackOnVideoTaskRunner,
 426                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 427                           CrossThreadBindRepeating(std::move(callback))));
 428 }
 429
 430 void MediaStreamVideoTrack::FrameDeliverer::AddEncodedCallbackOnVideoTaskRunner(
 431     VideoSinkId id,
 432     EncodedVideoFrameInternalCallback callback) {
 433   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 434   encoded_callbacks_.insert(id, std::move(callback));
 435 }
 436
 437 void MediaStreamVideoTrack::FrameDeliverer::RemoveCallback(VideoSinkId id) {
 438   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 439   PostCrossThreadTask(
 440       *video_task_runner_, FROM_HERE,
 441       CrossThreadBindOnce(&FrameDeliverer::RemoveCallbackOnVideoTaskRunner,
 442                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 443                           main_render_task_runner_));
 444 }
 445
 446 void MediaStreamVideoTrack::FrameDeliverer::RemoveCallbackOnVideoTaskRunner(
 447     VideoSinkId id,
 448     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
 449   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 450   auto* it = callbacks_.begin();
 451   for (; it != callbacks_.end(); ++it) {
 452     if (it->id == id) {
 453       // Callback destruction needs to happen on the specified task runner.
 454       PostCrossThreadTask(
 455           *task_runner, FROM_HERE,
 456           CrossThreadBindOnce(
 457               [](VideoCaptureDeliverFrameInternalCallback frame,
 458                  VideoCaptureNotifyFrameDroppedInternalCallback dropped) {},
 459               std::move(it->deliver_frame),
 460               std::move(it->notify_frame_dropped)));
 461       callbacks_.erase(it);
 462       return;
 463     }
 464   }
 465 }
 466
 467 void MediaStreamVideoTrack::FrameDeliverer::RemoveEncodedCallback(
 468     VideoSinkId id) {
 469   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 470   PostCrossThreadTask(
 471       *video_task_runner_, FROM_HERE,
 472       CrossThreadBindOnce(
 473           &FrameDeliverer::RemoveEncodedCallbackOnVideoTaskRunner,
 474           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 475           main_render_task_runner_));
 476 }
 477
 478 void MediaStreamVideoTrack::FrameDeliverer::
 479     RemoveEncodedCallbackOnVideoTaskRunner(
 480         VideoSinkId id,
 481         const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
 482   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 483
 484   // Callback destruction needs to happen on the specified task runner.
 485   auto it = encoded_callbacks_.find(id);
 486   if (it == encoded_callbacks_.end())
 487     return;
 488   PostCrossThreadTask(
 489       *task_runner, FROM_HERE,
 490       CrossThreadBindOnce([](EncodedVideoFrameInternalCallback callback) {},
 491                           std::move(it->value)));
 492   encoded_callbacks_.erase(it);
 493 }
 494
 495 void MediaStreamVideoTrack::FrameDeliverer::SetEnabled(bool enabled,
 496                                                        bool await_key_frame) {
 497   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 498   PostCrossThreadTask(
 499       *video_task_runner_, FROM_HERE,
 500       CrossThreadBindOnce(&FrameDeliverer::SetEnabledOnVideoTaskRunner,
 501                           WrapRefCounted(this), enabled, await_key_frame));
 502 }
 503
 504 void MediaStreamVideoTrack::FrameDeliverer::SetEnabledOnVideoTaskRunner(
 505     bool enabled,
 506     bool await_key_frame) {
 507   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 508   if (enabled != enabled_) {
 509     enabled_ = enabled;
 510     emit_frame_drop_events_ = true;
 511   }
 512   if (enabled_) {
 513     black_frame_ = nullptr;
 514     await_next_key_frame_ = await_key_frame;
 515   }
 516 }
 517
 518 void MediaStreamVideoTrack::FrameDeliverer::SetIsRefreshingForMinFrameRate(
 519     bool is_refreshing_for_min_frame_rate) {
 520   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 521   PostCrossThreadTask(
 522       *video_task_runner_, FROM_HERE,
 523       CrossThreadBindOnce(
 524           &FrameDeliverer::SetIsRefreshingForMinFrameRateOnVideoTaskRunner,
 525           WrapRefCounted(this), is_refreshing_for_min_frame_rate));
 526 }
 527
 528 void MediaStreamVideoTrack::FrameDeliverer::AddSubCaptureTargetVersionCallback(
 529     uint32_t sub_capture_target_version,
 530     base::OnceClosure callback) {
 531   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 532
 533   PostCrossThreadTask(
 534       *video_task_runner_, FROM_HERE,
 535       CrossThreadBindOnce(
 536           &FrameDeliverer::AddSubCaptureTargetVersionCallbackOnVideoTaskRunner,
 537           WrapRefCounted(this), sub_capture_target_version,
 538           CrossThreadBindOnce(std::move(callback))));
 539 }
 540
 541 void MediaStreamVideoTrack::FrameDeliverer::
 542     RemoveSubCaptureTargetVersionCallback(uint32_t sub_capture_target_version) {
 543   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 544
 545   PostCrossThreadTask(
 546       *video_task_runner_, FROM_HERE,
 547       CrossThreadBindOnce(
 548           &FrameDeliverer::
 549               RemoveSubCaptureTargetVersionCallbackOnVideoTaskRunner,
 550           WrapRefCounted(this), sub_capture_target_version));
 551 }
 552
 553 void MediaStreamVideoTrack::FrameDeliverer::
 554     SetIsRefreshingForMinFrameRateOnVideoTaskRunner(
 555         bool is_refreshing_for_min_frame_rate) {
 556   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 557   is_refreshing_for_min_frame_rate_ = is_refreshing_for_min_frame_rate;
 558 }
 559
 560 void MediaStreamVideoTrack::FrameDeliverer::
 561     AddSubCaptureTargetVersionCallbackOnVideoTaskRunner(
 562         uint32_t sub_capture_target_version,
 563         WTF::CrossThreadOnceClosure callback) {
 564   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 565   DCHECK(!base::Contains(sub_capture_target_version_callbacks_,
 566                          sub_capture_target_version));
 567
 568   sub_capture_target_version_callbacks_.Set(sub_capture_target_version,
 569                                             std::move(callback));
 570 }
 571
 572 void MediaStreamVideoTrack::FrameDeliverer::
 573     RemoveSubCaptureTargetVersionCallbackOnVideoTaskRunner(
 574         uint32_t sub_capture_target_version) {
 575   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 576
 577   // Note: Might or might not be here, depending on whether a later crop
 578   // version has already been observed or not.
 579   sub_capture_target_version_callbacks_.erase(sub_capture_target_version);
 580 }
 581
 582 void MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnVideoTaskRunner(
 583     scoped_refptr<media::VideoFrame> frame,
 584     base::TimeTicks estimated_capture_time) {
 585   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 586
 587   frame_drop_log_state_ = FrameDropLogState();
 588
 589   // TODO(crbug.com/1369085): Understand why we sometimes see old
 590   // sub-capture-target versions.
 591   if (frame->metadata().sub_capture_target_version !=
 592       sub_capture_target_version_) {
 593     OnFrameDroppedOnVideoTaskRunner(
 594         media::VideoCaptureFrameDropReason::kSubCaptureTargetVersionNotCurrent);
 595     return;
 596   }
 597
 598   if (!enabled_ && emit_frame_drop_events_) {
 599     emit_frame_drop_events_ = false;
 600     LogFrameDroppedOnVideoTaskRunner(
 601         media::VideoCaptureFrameDropReason::
 602             kVideoTrackFrameDelivererNotEnabledReplacingWithBlackFrame);
 603   }
 604   scoped_refptr<media::VideoFrame> video_frame;
 605   if (enabled_ || frame->metadata().end_of_stream) {
 606     video_frame = std::move(frame);
 607     ++deliverable_frames_;
 608   } else {
 609     // When disabled, a black video frame is passed along instead. The original
 610     // frames are dropped.
 611     video_frame = GetBlackFrame(*frame);
 612   }
 613   for (const auto& entry : callbacks_) {
 614     entry.deliver_frame.Run(video_frame, estimated_capture_time);
 615   }
 616   // The delay on refresh timer is reset each time a frame is received so that
 617   // it will not fire for at least an additional period. This means refresh
 618   // frames will only be requested when the source has halted delivery (e.g., a
 619   // screen capturer stops sending frames because the screen is not being
 620   // updated).
 621   if (is_refreshing_for_min_frame_rate_) {
 622     PostCrossThreadTask(
 623         *main_render_task_runner_, FROM_HERE,
 624         CrossThreadBindOnce(&MediaStreamVideoTrack::ResetRefreshTimer,
 625                             media_stream_video_track_));
 626   }
 627 }
 628
 629 void MediaStreamVideoTrack::FrameDeliverer::OnFrameDroppedOnVideoTaskRunner(
 630     media::VideoCaptureFrameDropReason reason) {
 631   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 632   DVLOG(1) << __func__;
 633   LogFrameDroppedOnVideoTaskRunner(reason);
 634   if (enabled_) {
 635     if (reason == media::VideoCaptureFrameDropReason::
 636                       kResolutionAdapterFrameRateIsHigherThanRequested) {
 637       ++discarded_frames_;
 638     } else {
 639       ++dropped_frames_;
 640     }
 641   }
 642   // Notify sinks that care about frame drops, i.e. WebRTC.
 643   for (const auto& entry : callbacks_) {
 644     entry.notify_frame_dropped.Run(reason);
 645   }
 646 }
 647
 648 void MediaStreamVideoTrack::FrameDeliverer::LogFrameDroppedOnVideoTaskRunner(
 649     media::VideoCaptureFrameDropReason reason) {
 650   MaybeEmitFrameDropLogMessage(reason);
 651
 652   if (reason == frame_drop_log_state_.drop_reason) {
 653     if (frame_drop_log_state_.max_log_count_exceeded) {
 654       return;
 655     }
 656
 657     if (++frame_drop_log_state_.drop_count >
 658         kMaxConsecutiveFrameDropForSameReasonCount) {
 659       frame_drop_log_state_.max_log_count_exceeded = true;
 660       return;
 661     }
 662   } else {
 663     frame_drop_log_state_ = FrameDropLogState(reason);
 664   }
 665
 666   LogVideoFrameDropUMA(reason, stream_type_);
 667 }
 668
 669 void MediaStreamVideoTrack::FrameDeliverer::MaybeEmitFrameDropLogMessage(
 670     media::VideoCaptureFrameDropReason reason) {
 671   using Type = std::underlying_type<media::VideoCaptureFrameDropReason>::type;
 672   static_assert(
 673       static_cast<Type>(media::VideoCaptureFrameDropReason::kMaxValue) <= 100,
 674       "Risk of memory overuse.");
 675
 676   static_assert(kMaxEmittedLogsForDroppedFramesBeforeSuppressing <
 677                     kFrequencyForSuppressedLogs,
 678                 "");
 679
 680   DCHECK_GE(static_cast<Type>(reason), 0);
 681   DCHECK_LE(reason, media::VideoCaptureFrameDropReason::kMaxValue);
 682
 683   int& occurrences = frame_drop_log_counters_[reason];
 684   if (++occurrences > kMaxEmittedLogsForDroppedFramesBeforeSuppressing &&
 685       occurrences % kFrequencyForSuppressedLogs != 0) {
 686     return;
 687   }
 688
 689   std::ostringstream string_stream;
 690   string_stream << "Frame dropped with reason code "
 691                 << static_cast<Type>(reason) << ".";
 692   if (occurrences == kMaxEmittedLogsForDroppedFramesBeforeSuppressing) {
 693     string_stream << " Additional logs will be partially suppressed.";
 694   }
 695
 696   // EmitLogMessage() unless overridden by testing.
 697   emit_log_message_.Run(string_stream.str());
 698 }
 699
 700 void MediaStreamVideoTrack::FrameDeliverer::SetEmitLogMessage(
 701     base::RepeatingCallback<void(const std::string&)> emit_log_message) {
 702   emit_log_message_ = std::move(emit_log_message);
 703 }
 704
 705 void MediaStreamVideoTrack::FrameDeliverer::EmitLogMessage(
 706     const std::string& message) {
 707   PostCrossThreadTask(*main_render_task_runner_, FROM_HERE,
 708                       CrossThreadBindOnce(&MediaStreamVideoSource::OnLog,
 709                                           media_stream_video_source_, message));
 710 }
 711
 712 void MediaStreamVideoTrack::FrameDeliverer::
 713     DeliverEncodedVideoFrameOnVideoTaskRunner(
 714         scoped_refptr<EncodedVideoFrame> frame,
 715         base::TimeTicks estimated_capture_time) {
 716   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 717   if (!enabled_) {
 718     return;
 719   }
 720   if (await_next_key_frame_ && !frame->IsKeyFrame()) {
 721     return;
 722   }
 723   await_next_key_frame_ = false;
 724   for (const auto& entry : encoded_callbacks_.Values()) {
 725     entry.Run(frame, estimated_capture_time);
 726   }
 727 }
 728
 729 void MediaStreamVideoTrack::FrameDeliverer::
 730     NewSubCaptureTargetVersionOnVideoTaskRunner(
 731         uint32_t sub_capture_target_version) {
 732   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 733   DCHECK_GT(sub_capture_target_version, sub_capture_target_version_);
 734
 735   sub_capture_target_version_ = sub_capture_target_version;
 736
 737   Vector<uint32_t> to_be_removed_keys;
 738   for (auto& iter : sub_capture_target_version_callbacks_) {
 739     if (iter.key > sub_capture_target_version) {
 740       continue;
 741     }
 742     std::move(iter.value).Run();
 743     to_be_removed_keys.push_back(iter.key);
 744   }
 745   sub_capture_target_version_callbacks_.RemoveAll(to_be_removed_keys);
 746 }
 747
 748 scoped_refptr<media::VideoFrame>
 749 MediaStreamVideoTrack::FrameDeliverer::GetBlackFrame(
 750     const media::VideoFrame& reference_frame) {
 751   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 752   if (!black_frame_.get() ||
 753       black_frame_->natural_size() != reference_frame.natural_size()) {
 754     black_frame_ =
 755         media::VideoFrame::CreateBlackFrame(reference_frame.natural_size());
 756   }
 757
 758   // Wrap |black_frame_| so we get a fresh timestamp we can modify. Frames
 759   // returned from this function may still be in use.
 760   scoped_refptr<media::VideoFrame> wrapped_black_frame =
 761       media::VideoFrame::WrapVideoFrame(black_frame_, black_frame_->format(),
 762                                         black_frame_->visible_rect(),
 763                                         black_frame_->natural_size());
 764   if (!wrapped_black_frame)
 765     return nullptr;
 766
 767   wrapped_black_frame->set_timestamp(reference_frame.timestamp());
 768   wrapped_black_frame->metadata().reference_time =
 769       reference_frame.metadata().reference_time;
 770
 771   return wrapped_black_frame;
 772 }
 773
 774 // static
 775 WebMediaStreamTrack MediaStreamVideoTrack::CreateVideoTrack(
 776     MediaStreamVideoSource* source,
 777     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 778     bool enabled) {
 779   auto* component = MakeGarbageCollected<MediaStreamComponentImpl>(
 780       source->Owner(), std::make_unique<MediaStreamVideoTrack>(
 781                            source, std::move(callback), enabled));
 782   return WebMediaStreamTrack(component);
 783 }
 784
 785 // static
 786 WebMediaStreamTrack MediaStreamVideoTrack::CreateVideoTrack(
 787     MediaStreamVideoSource* source,
 788     const VideoTrackAdapterSettings& adapter_settings,
 789     const absl::optional<bool>& noise_reduction,
 790     bool is_screencast,
 791     const absl::optional<double>& min_frame_rate,
 792     const absl::optional<double>& pan,
 793     const absl::optional<double>& tilt,
 794     const absl::optional<double>& zoom,
 795     bool pan_tilt_zoom_allowed,
 796     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 797     bool enabled) {
 798   WebMediaStreamTrack track;
 799   auto* component = MakeGarbageCollected<MediaStreamComponentImpl>(
 800       source->Owner(),
 801       std::make_unique<MediaStreamVideoTrack>(
 802           source, adapter_settings, noise_reduction, is_screencast,
 803           min_frame_rate, pan, tilt, zoom, pan_tilt_zoom_allowed,
 804           std::move(callback), enabled));
 805   return WebMediaStreamTrack(component);
 806 }
 807
 808 // static
 809 MediaStreamVideoTrack* MediaStreamVideoTrack::From(
 810     const MediaStreamComponent* component) {
 811   if (!component ||
 812       component->GetSourceType() != MediaStreamSource::kTypeVideo) {
 813     return nullptr;
 814   }
 815
 816   return static_cast<MediaStreamVideoTrack*>(component->GetPlatformTrack());
 817 }
 818
 819 MediaStreamVideoTrack::MediaStreamVideoTrack(
 820     MediaStreamVideoSource* source,
 821     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 822     bool enabled)
 823     : MediaStreamTrackPlatform(true),
 824       is_screencast_(false),
 825       source_(source->GetWeakPtr()) {
 826   frame_deliverer_ =
 827       base::MakeRefCounted<MediaStreamVideoTrack::FrameDeliverer>(
 828           source->GetTaskRunner(), source->video_task_runner(),
 829           weak_factory_.GetWeakPtr(), source->GetWeakPtr(), enabled,
 830           source->GetSubCaptureTargetVersion());
 831   source->AddTrack(
 832       this, VideoTrackAdapterSettings(),
 833       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 834           &MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnVideoTaskRunner,
 835           frame_deliverer_)),
 836       ConvertToBaseRepeatingCallback(
 837           CrossThreadBindRepeating(&MediaStreamVideoTrack::FrameDeliverer::
 838                                        OnFrameDroppedOnVideoTaskRunner,
 839                                    frame_deliverer_)),
 840       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 841           &MediaStreamVideoTrack::FrameDeliverer::
 842               DeliverEncodedVideoFrameOnVideoTaskRunner,
 843           frame_deliverer_)),
 844       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 845           &MediaStreamVideoTrack::FrameDeliverer::
 846               NewSubCaptureTargetVersionOnVideoTaskRunner,
 847           frame_deliverer_)),
 848       base::BindPostTaskToCurrentDefault(WTF::BindRepeating(
 849           &MediaStreamVideoTrack::SetSizeAndComputedFrameRate,
 850           weak_factory_.GetWeakPtr())),
 851       base::BindPostTaskToCurrentDefault(
 852           WTF::BindRepeating(&MediaStreamVideoTrack::set_computed_source_format,
 853                              weak_factory_.GetWeakPtr())),
 854       std::move(callback));
 855 }
 856
 857 MediaStreamVideoTrack::MediaStreamVideoTrack(
 858     MediaStreamVideoSource* source,
 859     const VideoTrackAdapterSettings& adapter_settings,
 860     const absl::optional<bool>& noise_reduction,
 861     bool is_screen_cast,
 862     const absl::optional<double>& min_frame_rate,
 863     const absl::optional<double>& pan,
 864     const absl::optional<double>& tilt,
 865     const absl::optional<double>& zoom,
 866     bool pan_tilt_zoom_allowed,
 867     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 868     bool enabled)
 869     : MediaStreamTrackPlatform(true),
 870       adapter_settings_(adapter_settings),
 871       noise_reduction_(noise_reduction),
 872       is_screencast_(is_screen_cast),
 873       min_frame_rate_(min_frame_rate),
 874       pan_(pan),
 875       tilt_(tilt),
 876       zoom_(zoom),
 877       pan_tilt_zoom_allowed_(pan_tilt_zoom_allowed),
 878       source_(source->GetWeakPtr()) {
 879   frame_deliverer_ =
 880       base::MakeRefCounted<MediaStreamVideoTrack::FrameDeliverer>(
 881           source->GetTaskRunner(), source->video_task_runner(),
 882           weak_factory_.GetWeakPtr(), source->GetWeakPtr(), enabled,
 883           source->GetSubCaptureTargetVersion());
 884   source->AddTrack(
 885       this, adapter_settings,
 886       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 887           &MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnVideoTaskRunner,
 888           frame_deliverer_)),
 889       ConvertToBaseRepeatingCallback(
 890           CrossThreadBindRepeating(&MediaStreamVideoTrack::FrameDeliverer::
 891                                        OnFrameDroppedOnVideoTaskRunner,
 892                                    frame_deliverer_)),
 893       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 894           &MediaStreamVideoTrack::FrameDeliverer::
 895               DeliverEncodedVideoFrameOnVideoTaskRunner,
 896           frame_deliverer_)),
 897       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 898           &MediaStreamVideoTrack::FrameDeliverer::
 899               NewSubCaptureTargetVersionOnVideoTaskRunner,
 900           frame_deliverer_)),
 901       base::BindPostTaskToCurrentDefault(WTF::BindRepeating(
 902           &MediaStreamVideoTrack::SetSizeAndComputedFrameRate,
 903           weak_factory_.GetWeakPtr())),
 904       base::BindPostTaskToCurrentDefault(
 905           WTF::BindRepeating(&MediaStreamVideoTrack::set_computed_source_format,
 906                              weak_factory_.GetWeakPtr())),
 907       std::move(callback));
 908 }
 909
 910 MediaStreamVideoTrack::~MediaStreamVideoTrack() {
 911   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 912   DCHECK(sinks_.empty());
 913   DCHECK(encoded_sinks_.empty());
 914   Stop();
 915   DVLOG(3) << "~MediaStreamVideoTrack()";
 916 }
 917
 918 std::unique_ptr<MediaStreamTrackPlatform>
 919 MediaStreamVideoTrack::CreateFromComponent(
 920     const MediaStreamComponent* component,
 921     const String& id) {
 922   MediaStreamSource* source = component->Source();
 923   DCHECK_EQ(source->GetType(), MediaStreamSource::kTypeVideo);
 924   MediaStreamVideoSource* native_source =
 925       MediaStreamVideoSource::GetVideoSource(source);
 926   DCHECK(native_source);
 927   MediaStreamVideoTrack* original_track =
 928       MediaStreamVideoTrack::From(component);
 929   DCHECK(original_track);
 930   return std::make_unique<MediaStreamVideoTrack>(
 931       native_source, original_track->adapter_settings(),
 932       original_track->noise_reduction(), original_track->is_screencast(),
 933       original_track->min_frame_rate(), original_track->pan(),
 934       original_track->tilt(), original_track->zoom(),
 935       original_track->pan_tilt_zoom_allowed(),
 936       MediaStreamVideoSource::ConstraintsOnceCallback(), component->Enabled());
 937 }
 938
 939 static void AddSinkInternal(Vector<WebMediaStreamSink*>* sinks,
 940                             WebMediaStreamSink* sink) {
 941   DCHECK(!base::Contains(*sinks, sink));
 942   sinks->push_back(sink);
 943 }
 944
 945 static void RemoveSinkInternal(Vector<WebMediaStreamSink*>* sinks,
 946                                WebMediaStreamSink* sink) {
 947   auto** it = base::ranges::find(*sinks, sink);
 948   DCHECK(it != sinks->end());
 949   sinks->erase(it);
 950 }
 951
 952 void MediaStreamVideoTrack::AddSink(
 953     WebMediaStreamSink* sink,
 954     const VideoCaptureDeliverFrameCB& callback,
 955     MediaStreamVideoSink::IsSecure is_secure,
 956     MediaStreamVideoSink::UsesAlpha uses_alpha) {
 957   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 958   AddSinkInternal(&sinks_, sink);
 959   frame_deliverer_->AddCallback(sink, callback);
 960   secure_tracker_.Add(sink, is_secure == MediaStreamVideoSink::IsSecure::kYes);
 961   if (uses_alpha == MediaStreamVideoSink::UsesAlpha::kDefault) {
 962     alpha_using_sinks_.insert(sink);
 963   } else if (uses_alpha == MediaStreamVideoSink::UsesAlpha::kNo) {
 964     alpha_discarding_sinks_.insert(sink);
 965   }
 966
 967   // Ensure sink gets told about any constraints set.
 968   sink->OnVideoConstraintsChanged(min_frame_rate_,
 969                                   adapter_settings_.max_frame_rate());
 970
 971   // Request source to deliver a frame because a new sink is added.
 972   if (!source_)
 973     return;
 974   UpdateSourceHasConsumers();
 975   RequestRefreshFrame();
 976   source_->UpdateCapturingLinkSecure(this,
 977                                      secure_tracker_.is_capturing_secure());
 978
 979   source_->UpdateCanDiscardAlpha();
 980
 981   if (is_screencast_)
 982     StartTimerForRequestingFrames();
 983 }
 984
 985 bool MediaStreamVideoTrack::UsingAlpha() {
 986   // Alpha can't be discarded if any sink uses alpha, or if the only sinks
 987   // connected are kDependsOnOtherSinks.
 988   bool only_sinks_with_alpha_depending_on_other_sinks =
 989       !sinks_.empty() && alpha_using_sinks_.empty() &&
 990       alpha_discarding_sinks_.empty();
 991   return !alpha_using_sinks_.empty() ||
 992          only_sinks_with_alpha_depending_on_other_sinks;
 993 }
 994
 995 void MediaStreamVideoTrack::SetSinkNotifyFrameDroppedCallback(
 996     WebMediaStreamSink* sink,
 997     const VideoCaptureNotifyFrameDroppedCB& callback) {
 998   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 999   DVLOG(1) << __func__;
1000   frame_deliverer_->SetNotifyFrameDroppedCallback(sink, callback);
1001 }
1002
1003 void MediaStreamVideoTrack::AddEncodedSink(WebMediaStreamSink* sink,
1004                                            EncodedVideoFrameCB callback) {
1005   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1006   AddSinkInternal(&encoded_sinks_, sink);
1007   frame_deliverer_->AddEncodedCallback(sink, std::move(callback));
1008   if (source_)
1009     source_->UpdateNumEncodedSinks();
1010   UpdateSourceHasConsumers();
1011 }
1012
1013 void MediaStreamVideoTrack::RemoveSink(WebMediaStreamSink* sink) {
1014   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1015   RemoveSinkInternal(&sinks_, sink);
1016   alpha_using_sinks_.erase(sink);
1017   alpha_discarding_sinks_.erase(sink);
1018   frame_deliverer_->RemoveCallback(sink);
1019   secure_tracker_.Remove(sink);
1020   if (!source_)
1021     return;
1022   UpdateSourceHasConsumers();
1023   source_->UpdateCapturingLinkSecure(this,
1024                                      secure_tracker_.is_capturing_secure());
1025
1026   source_->UpdateCanDiscardAlpha();
1027   // Restart the timer with existing sinks.
1028   if (is_screencast_)
1029     StartTimerForRequestingFrames();
1030 }
1031
1032 void MediaStreamVideoTrack::RemoveEncodedSink(WebMediaStreamSink* sink) {
1033   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1034   RemoveSinkInternal(&encoded_sinks_, sink);
1035   frame_deliverer_->RemoveEncodedCallback(sink);
1036   if (source_)
1037     source_->UpdateNumEncodedSinks();
1038   UpdateSourceHasConsumers();
1039 }
1040
1041 void MediaStreamVideoTrack::UpdateSourceHasConsumers() {
1042   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1043   if (!source_)
1044     return;
1045   bool has_consumers = !sinks_.empty() || !encoded_sinks_.empty();
1046   source_->UpdateHasConsumers(this, has_consumers);
1047 }
1048
1049 void MediaStreamVideoTrack::SetEnabled(bool enabled) {
1050   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1051   // If enabled, encoded sinks exist and the source supports encoded output, we
1052   // need a new keyframe from the source as we may have dropped data making the
1053   // stream undecodable.
1054   bool maybe_await_key_frame = false;
1055   if (enabled && source_ && source_->SupportsEncodedOutput() &&
1056       !encoded_sinks_.empty()) {
1057     RequestRefreshFrame();
1058     maybe_await_key_frame = true;
1059   }
1060   frame_deliverer_->SetEnabled(enabled, maybe_await_key_frame);
1061   for (auto* sink : sinks_)
1062     sink->OnEnabledChanged(enabled);
1063   for (auto* encoded_sink : encoded_sinks_)
1064     encoded_sink->OnEnabledChanged(enabled);
1065 }
1066
1067 size_t MediaStreamVideoTrack::CountSinks() const {
1068   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1069   return sinks_.size();
1070 }
1071
1072 size_t MediaStreamVideoTrack::CountEncodedSinks() const {
1073   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1074   return encoded_sinks_.size();
1075 }
1076
1077 void MediaStreamVideoTrack::SetContentHint(
1078     WebMediaStreamTrack::ContentHintType content_hint) {
1079   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1080   for (auto* sink : sinks_)
1081     sink->OnContentHintChanged(content_hint);
1082   for (auto* encoded_sink : encoded_sinks_)
1083     encoded_sink->OnContentHintChanged(content_hint);
1084 }
1085
1086 void MediaStreamVideoTrack::StopAndNotify(base::OnceClosure callback) {
1087   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1088   if (source_) {
1089     source_->RemoveTrack(this, std::move(callback));
1090     source_ = nullptr;
1091   } else if (callback) {
1092     std::move(callback).Run();
1093   }
1094   OnReadyStateChanged(WebMediaStreamSource::kReadyStateEnded);
1095   refresh_timer_.Stop();
1096 }
1097
1098 void MediaStreamVideoTrack::GetSettings(
1099     MediaStreamTrackPlatform::Settings& settings) const {
1100   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1101   if (!source_)
1102     return;
1103
1104   if (width_ && height_) {
1105     settings.width = width_;
1106     settings.height = height_;
1107     settings.aspect_ratio = static_cast<double>(width_) / height_;
1108   }
1109
1110   if (absl::optional<media::VideoCaptureFormat> format =
1111           source_->GetCurrentFormat()) {
1112     // For local capture-based tracks, the frame rate returned by
1113     // MediaStreamTrack.getSettings() must be the configured frame rate. In case
1114     // of frame rate decimation, the configured frame rate is the decimated
1115     // frame rate (i.e., the adapter frame rate). If there is no decimation, the
1116     // configured frame rate is the frame rate reported by the device.
1117     // Decimation occurs only when the adapter frame rate is lower than the
1118     // device frame rate.
1119     absl::optional<double> adapter_frame_rate =
1120         adapter_settings_.max_frame_rate();
1121     settings.frame_rate =
1122         (!adapter_frame_rate || *adapter_frame_rate > format->frame_rate)
1123             ? format->frame_rate
1124             : *adapter_frame_rate;
1125   } else {
1126     // For other tracks, use the computed frame rate reported via
1127     // SetSizeAndComputedFrameRate().
1128     if (computed_frame_rate_)
1129       settings.frame_rate = *computed_frame_rate_;
1130   }
1131
1132   settings.facing_mode = ToPlatformFacingMode(
1133       static_cast<mojom::blink::FacingMode>(source_->device().video_facing));
1134   settings.resize_mode = WebString::FromASCII(std::string(
1135       adapter_settings().target_size() ? WebMediaStreamTrack::kResizeModeRescale
1136                                        : WebMediaStreamTrack::kResizeModeNone));
1137   if (source_->device().display_media_info) {
1138     const auto& info = source_->device().display_media_info;
1139     settings.display_surface = info->display_surface;
1140     settings.logical_surface = info->logical_surface;
1141     settings.cursor = info->cursor;
1142   }
1143 }
1144
1145 MediaStreamTrackPlatform::VideoFrameStats
1146 MediaStreamVideoTrack::GetVideoFrameStats() const {
1147   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1148   MediaStreamTrackPlatform::VideoFrameStats stats;
1149   stats.deliverable_frames = frame_deliverer_->deliverable_frames();
1150   stats.discarded_frames = frame_deliverer_->discarded_frames();
1151   stats.dropped_frames = frame_deliverer_->dropped_frames();
1152   return stats;
1153 }
1154
1155 MediaStreamTrackPlatform::CaptureHandle
1156 MediaStreamVideoTrack::GetCaptureHandle() {
1157   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1158
1159   MediaStreamTrackPlatform::CaptureHandle capture_handle;
1160
1161   if (!source_) {
1162     return capture_handle;
1163   }
1164
1165   const MediaStreamDevice& device = source_->device();
1166   if (!device.display_media_info) {
1167     return capture_handle;
1168   }
1169   const media::mojom::DisplayMediaInformationPtr& info =
1170       device.display_media_info;
1171
1172   if (!info->capture_handle) {
1173     return capture_handle;
1174   }
1175
1176   if (!info->capture_handle->origin.opaque()) {
1177     capture_handle.origin =
1178         String::FromUTF8(info->capture_handle->origin.Serialize());
1179   }
1180   capture_handle.handle =
1181       WebString::FromUTF16(info->capture_handle->capture_handle);
1182
1183   return capture_handle;
1184 }
1185
1186 void MediaStreamVideoTrack::AddSubCaptureTargetVersionCallback(
1187     uint32_t sub_capture_target_version,
1188     base::OnceClosure callback) {
1189   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1190
1191   frame_deliverer_->AddSubCaptureTargetVersionCallback(
1192       sub_capture_target_version,
1193       base::BindPostTask(base::SingleThreadTaskRunner::GetCurrentDefault(),
1194                          std::move(callback)));
1195 }
1196
1197 void MediaStreamVideoTrack::RemoveSubCaptureTargetVersionCallback(
1198     uint32_t sub_capture_target_version) {
1199   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1200
1201   frame_deliverer_->RemoveSubCaptureTargetVersionCallback(
1202       sub_capture_target_version);
1203 }
1204
1205 void MediaStreamVideoTrack::OnReadyStateChanged(
1206     WebMediaStreamSource::ReadyState state) {
1207   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1208
1209   // Copy the vectors first, since sinks might DisconnectFromTrack() and
1210   // invalidate iterators.
1211
1212   Vector<WebMediaStreamSink*> sinks_copy(sinks_);
1213   for (auto* sink : sinks_copy)
1214     sink->OnReadyStateChanged(state);
1215
1216   Vector<WebMediaStreamSink*> encoded_sinks_copy(encoded_sinks_);
1217   for (auto* encoded_sink : encoded_sinks_copy)
1218     encoded_sink->OnReadyStateChanged(state);
1219 }
1220
1221 void MediaStreamVideoTrack::SetMinimumFrameRate(double min_frame_rate) {
1222   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1223   min_frame_rate_ = min_frame_rate;
1224 }
1225
1226 void MediaStreamVideoTrack::SetTrackAdapterSettings(
1227     const VideoTrackAdapterSettings& settings) {
1228   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1229   adapter_settings_ = settings;
1230 }
1231
1232 void MediaStreamVideoTrack::NotifyConstraintsConfigurationComplete() {
1233   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1234   for (auto* sink : sinks_) {
1235     sink->OnVideoConstraintsChanged(min_frame_rate_,
1236                                     adapter_settings_.max_frame_rate());
1237   }
1238
1239   if (is_screencast_) {
1240     StartTimerForRequestingFrames();
1241   }
1242 }
1243
1244 media::VideoCaptureFormat MediaStreamVideoTrack::GetComputedSourceFormat() {
1245   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1246   return computed_source_format_;
1247 }
1248
1249 void MediaStreamVideoTrack::OnSinkDroppedFrame(
1250     media::VideoCaptureFrameDropReason reason) {
1251   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1252   if (!source_) {
1253     return;
1254   }
1255   PostCrossThreadTask(
1256       *source_->video_task_runner(), FROM_HERE,
1257       CrossThreadBindOnce(&FrameDeliverer::LogFrameDroppedOnVideoTaskRunner,
1258                           frame_deliverer_, reason));
1259 }
1260
1261 void MediaStreamVideoTrack::SetEmitLogMessageForTesting(
1262     base::RepeatingCallback<void(const std::string&)> emit_log_message) {
1263   frame_deliverer_->SetEmitLogMessage(std::move(emit_log_message));
1264 }
1265
1266 void MediaStreamVideoTrack::StartTimerForRequestingFrames() {
1267   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1268
1269   // Find the maximum of all the required min frames per second in the attached
1270   // sinks.
1271   double required_min_fps = 0;
1272   for (auto* web_sink : sinks_) {
1273     auto* sink = static_cast<MediaStreamVideoSink*>(web_sink);
1274     required_min_fps =
1275         std::max(required_min_fps, sink->GetRequiredMinFramesPerSec());
1276   }
1277
1278   base::TimeDelta refresh_interval = ComputeRefreshIntervalFromBounds(
1279       base::Hertz(required_min_fps), min_frame_rate(), max_frame_rate());
1280
1281   if (refresh_interval.is_max()) {
1282     refresh_timer_.Stop();
1283     frame_deliverer_->SetIsRefreshingForMinFrameRate(false);
1284   } else {
1285     DVLOG(1) << "Starting frame refresh timer with interval "
1286              << refresh_interval.InMillisecondsF() << " ms.";
1287     refresh_timer_.Start(FROM_HERE, refresh_interval, this,
1288                          &MediaStreamVideoTrack::RequestRefreshFrame);
1289     frame_deliverer_->SetIsRefreshingForMinFrameRate(true);
1290   }
1291 }
1292
1293 void MediaStreamVideoTrack::RequestRefreshFrame() {
1294   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1295   if (source_)
1296     source_->RequestRefreshFrame();
1297 }
1298
1299 void MediaStreamVideoTrack::ResetRefreshTimer() {
1300   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
1301   if (refresh_timer_.IsRunning())
1302     refresh_timer_.Reset();
1303 }
1304
1305 }  // namespace blink