third_party/blink/renderer/modules/mediastream/media_stream_video_track.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
   6
   7 #include <string>
   8 #include <utility>
   9
  10 #include "base/bind.h"
  11 #include "base/containers/contains.h"
  12 #include "base/location.h"
  13 #include "base/macros.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "build/build_config.h"
  16 #include "media/base/bind_to_current_loop.h"
  17 #include "media/base/limits.h"
  18 #include "media/capture/video_capture_types.h"
  19 #include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h"
  20 #include "third_party/blink/public/web/web_local_frame.h"
  21 #include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_device.h"
  22 #include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
  23 #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
  24 #include "third_party/blink/renderer/platform/scheduler/public/thread.h"
  25 #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
  26 #include "third_party/blink/renderer/platform/wtf/vector.h"
  27
  28 namespace blink {
  29 namespace {
  30
  31 // A lower-bound for the refresh interval.
  32 constexpr base::TimeDelta kLowerBoundRefreshInterval =
  33     base::TimeDelta::FromHz(media::limits::kMaxFramesPerSecond);
  34
  35 // This alias mimics the definition of VideoCaptureDeliverFrameCB.
  36 using VideoCaptureDeliverFrameInternalCallback = WTF::CrossThreadFunction<void(
  37     scoped_refptr<media::VideoFrame> video_frame,
  38     std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
  39     base::TimeTicks estimated_capture_time)>;
  40
  41 // Mimics blink::EncodedVideoFrameCB
  42 using EncodedVideoFrameInternalCallback =
  43     WTF::CrossThreadFunction<void(scoped_refptr<EncodedVideoFrame> frame,
  44                                   base::TimeTicks estimated_capture_time)>;
  45
  46 base::TimeDelta ComputeRefreshIntervalFromBounds(
  47     const base::TimeDelta required_min_refresh_interval,
  48     const absl::optional<double>& min_frame_rate,
  49     const absl::optional<double>& max_frame_rate) {
  50   // Start with the default required refresh interval, and refine based on
  51   // constraints. If a minimum frameRate is provided, use that. Otherwise, use
  52   // the maximum frameRate if it happens to be less than the default.
  53   base::TimeDelta refresh_interval = required_min_refresh_interval;
  54   if (min_frame_rate.has_value())
  55     refresh_interval = base::TimeDelta::FromHz(*min_frame_rate);
  56
  57   if (max_frame_rate.has_value()) {
  58     refresh_interval =
  59         std::max(refresh_interval, base::TimeDelta::FromHz(*max_frame_rate));
  60   }
  61
  62   if (refresh_interval < kLowerBoundRefreshInterval)
  63     refresh_interval = kLowerBoundRefreshInterval;
  64
  65   return refresh_interval;
  66 }
  67
  68 }  // namespace
  69
  70 // MediaStreamVideoTrack::FrameDeliverer is a helper class used for registering
  71 // VideoCaptureDeliverFrameCB/EncodedVideoFrameCB callbacks on the main render
  72 // thread to receive video frames on the IO-thread. Frames are only delivered to
  73 // the sinks if the track is enabled. If the track is disabled, a black frame is
  74 // instead forwarded to the sinks at the same frame rate. A disabled track does
  75 // not forward data to encoded sinks.
  76 class MediaStreamVideoTrack::FrameDeliverer
  77     : public WTF::ThreadSafeRefCounted<FrameDeliverer> {
  78  public:
  79   using VideoSinkId = WebMediaStreamSink*;
  80
  81   FrameDeliverer(scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
  82                  base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track,
  83                  bool enabled);
  84
  85   // Sets whether the track is enabled or not. If getting enabled and encoded
  86   // output is enabled, the deliverer will wait until the next key frame before
  87   // it resumes producing encoded data.
  88   void SetEnabled(bool enabled, bool await_key_frame);
  89
  90   // Add |callback| to receive video frames on the IO-thread.
  91   // Must be called on the main render thread.
  92   void AddCallback(VideoSinkId id, VideoCaptureDeliverFrameCB callback);
  93
  94   // Add |callback| to receive encoded video frames on the IO-thread.
  95   // Must be called on the main render thread.
  96   void AddEncodedCallback(VideoSinkId id, EncodedVideoFrameCB callback);
  97
  98   // Removes |callback| associated with |id| from receiving video frames if |id|
  99   // has been added. It is ok to call RemoveCallback even if the |id| has not
 100   // been added. Note that the added callback will be reset on the main thread.
 101   // Must be called on the main render thread.
 102   void RemoveCallback(VideoSinkId id);
 103
 104   // Removes encoded callback associated with |id| from receiving video frames
 105   // if |id| has been added. It is ok to call RemoveEncodedCallback even if the
 106   // |id| has not been added. Note that the added callback will be reset on the
 107   // main thread. Must be called on the main render thread.
 108   void RemoveEncodedCallback(VideoSinkId id);
 109
 110   // Triggers all registered callbacks with |frame| and |estimated_capture_time|
 111   // as parameters. Must be called on the IO-thread.
 112   void DeliverFrameOnIO(
 113       scoped_refptr<media::VideoFrame> frame,
 114       std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
 115       base::TimeTicks estimated_capture_time);
 116
 117   // Triggers all encoded callbacks with |frame| and |estimated_capture_time|.
 118   // Must be called on the IO-thread.
 119   void DeliverEncodedVideoFrameOnIO(scoped_refptr<EncodedVideoFrame> frame,
 120                                     base::TimeTicks estimated_capture_time);
 121
 122   void SetIsRefreshingForMinFrameRate(bool is_refreshing_for_min_frame_rate);
 123
 124  private:
 125   friend class WTF::ThreadSafeRefCounted<FrameDeliverer>;
 126   virtual ~FrameDeliverer();
 127   void AddCallbackOnIO(VideoSinkId id,
 128                        VideoCaptureDeliverFrameInternalCallback callback);
 129   void RemoveCallbackOnIO(
 130       VideoSinkId id,
 131       const scoped_refptr<base::SingleThreadTaskRunner>& task_runner);
 132
 133   void AddEncodedCallbackOnIO(VideoSinkId id,
 134                               EncodedVideoFrameInternalCallback callback);
 135   void RemoveEncodedCallbackOnIO(
 136       VideoSinkId id,
 137       const scoped_refptr<base::SingleThreadTaskRunner>& task_runner);
 138
 139   void SetEnabledOnIO(bool enabled, bool await_key_frame);
 140
 141   void SetIsRefreshingForMinFrameRateOnIO(
 142       bool is_refreshing_for_min_frame_rate);
 143
 144   // Returns a black frame where the size and time stamp is set to the same as
 145   // as in |reference_frame|.
 146   scoped_refptr<media::VideoFrame> GetBlackFrame(
 147       const media::VideoFrame& reference_frame);
 148
 149   // Used to DCHECK that AddCallback and RemoveCallback are called on the main
 150   // Render Thread.
 151   THREAD_CHECKER(main_render_thread_checker_);
 152   const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
 153   // Can be null in testing.
 154   scoped_refptr<base::SingleThreadTaskRunner> main_render_task_runner_;
 155
 156   base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track_;
 157
 158   bool enabled_;
 159   scoped_refptr<media::VideoFrame> black_frame_;
 160   bool emit_frame_drop_events_;
 161
 162   using VideoIdCallbackPair =
 163       std::pair<VideoSinkId, VideoCaptureDeliverFrameInternalCallback>;
 164   Vector<VideoIdCallbackPair> callbacks_;
 165   HashMap<VideoSinkId, EncodedVideoFrameInternalCallback> encoded_callbacks_;
 166   bool await_next_key_frame_;
 167
 168   // This should only be accessed on the IO thread.
 169   bool is_refreshing_for_min_frame_rate_ = false;
 170
 171   DISALLOW_COPY_AND_ASSIGN(FrameDeliverer);
 172 };
 173
 174 MediaStreamVideoTrack::FrameDeliverer::FrameDeliverer(
 175     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
 176     base::WeakPtr<MediaStreamVideoTrack> media_stream_video_track,
 177     bool enabled)
 178     : io_task_runner_(std::move(io_task_runner)),
 179       media_stream_video_track_(media_stream_video_track),
 180       enabled_(enabled),
 181       emit_frame_drop_events_(true),
 182       await_next_key_frame_(false) {
 183   DCHECK(io_task_runner_.get());
 184
 185   WebLocalFrame* web_frame = WebLocalFrame::FrameForCurrentContext();
 186   if (web_frame) {
 187     main_render_task_runner_ =
 188         web_frame->GetTaskRunner(TaskType::kInternalMedia);
 189   }
 190 }
 191
 192 MediaStreamVideoTrack::FrameDeliverer::~FrameDeliverer() {
 193   DCHECK(callbacks_.IsEmpty());
 194 }
 195
 196 void MediaStreamVideoTrack::FrameDeliverer::AddCallback(
 197     VideoSinkId id,
 198     VideoCaptureDeliverFrameCB callback) {
 199   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 200   PostCrossThreadTask(
 201       *io_task_runner_, FROM_HERE,
 202       CrossThreadBindOnce(&FrameDeliverer::AddCallbackOnIO,
 203                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 204                           CrossThreadBindRepeating(std::move(callback))));
 205 }
 206
 207 void MediaStreamVideoTrack::FrameDeliverer::AddCallbackOnIO(
 208     VideoSinkId id,
 209     VideoCaptureDeliverFrameInternalCallback callback) {
 210   DCHECK(io_task_runner_->BelongsToCurrentThread());
 211   callbacks_.push_back(std::make_pair(id, std::move(callback)));
 212 }
 213
 214 void MediaStreamVideoTrack::FrameDeliverer::AddEncodedCallback(
 215     VideoSinkId id,
 216     EncodedVideoFrameCB callback) {
 217   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 218   PostCrossThreadTask(
 219       *io_task_runner_, FROM_HERE,
 220       CrossThreadBindOnce(&FrameDeliverer::AddEncodedCallbackOnIO,
 221                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 222                           CrossThreadBindRepeating(std::move(callback))));
 223 }
 224
 225 void MediaStreamVideoTrack::FrameDeliverer::AddEncodedCallbackOnIO(
 226     VideoSinkId id,
 227     EncodedVideoFrameInternalCallback callback) {
 228   DCHECK(io_task_runner_->BelongsToCurrentThread());
 229   encoded_callbacks_.insert(id, std::move(callback));
 230 }
 231
 232 void MediaStreamVideoTrack::FrameDeliverer::RemoveCallback(VideoSinkId id) {
 233   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 234   PostCrossThreadTask(
 235       *io_task_runner_, FROM_HERE,
 236       CrossThreadBindOnce(&FrameDeliverer::RemoveCallbackOnIO,
 237                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 238                           Thread::Current()->GetTaskRunner()));
 239 }
 240
 241 void MediaStreamVideoTrack::FrameDeliverer::RemoveCallbackOnIO(
 242     VideoSinkId id,
 243     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
 244   DCHECK(io_task_runner_->BelongsToCurrentThread());
 245   auto* it = callbacks_.begin();
 246   for (; it != callbacks_.end(); ++it) {
 247     if (it->first == id) {
 248       // Callback destruction needs to happen on the specified task runner.
 249       PostCrossThreadTask(
 250           *task_runner, FROM_HERE,
 251           CrossThreadBindOnce(
 252               [](VideoCaptureDeliverFrameInternalCallback callback) {},
 253               std::move(it->second)));
 254       callbacks_.erase(it);
 255       return;
 256     }
 257   }
 258 }
 259
 260 void MediaStreamVideoTrack::FrameDeliverer::RemoveEncodedCallback(
 261     VideoSinkId id) {
 262   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 263   PostCrossThreadTask(
 264       *io_task_runner_, FROM_HERE,
 265       CrossThreadBindOnce(&FrameDeliverer::RemoveEncodedCallbackOnIO,
 266                           WrapRefCounted(this), WTF::CrossThreadUnretained(id),
 267                           Thread::Current()->GetTaskRunner()));
 268 }
 269
 270 void MediaStreamVideoTrack::FrameDeliverer::RemoveEncodedCallbackOnIO(
 271     VideoSinkId id,
 272     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
 273   DCHECK(io_task_runner_->BelongsToCurrentThread());
 274
 275   // Callback destruction needs to happen on the specified task runner.
 276   auto it = encoded_callbacks_.find(id);
 277   if (it == encoded_callbacks_.end()) {
 278     return;
 279   }
 280   PostCrossThreadTask(
 281       *task_runner, FROM_HERE,
 282       CrossThreadBindOnce([](EncodedVideoFrameInternalCallback callback) {},
 283                           std::move(it->value)));
 284   encoded_callbacks_.erase(it);
 285 }
 286
 287 void MediaStreamVideoTrack::FrameDeliverer::SetEnabled(bool enabled,
 288                                                        bool await_key_frame) {
 289   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 290   PostCrossThreadTask(
 291       *io_task_runner_, FROM_HERE,
 292       CrossThreadBindOnce(&FrameDeliverer::SetEnabledOnIO, WrapRefCounted(this),
 293                           enabled, await_key_frame));
 294 }
 295
 296 void MediaStreamVideoTrack::FrameDeliverer::SetEnabledOnIO(
 297     bool enabled,
 298     bool await_key_frame) {
 299   DCHECK(io_task_runner_->BelongsToCurrentThread());
 300   if (enabled != enabled_) {
 301     enabled_ = enabled;
 302     emit_frame_drop_events_ = true;
 303   }
 304   if (enabled_) {
 305     black_frame_ = nullptr;
 306     await_next_key_frame_ = await_key_frame;
 307   }
 308 }
 309
 310 void MediaStreamVideoTrack::FrameDeliverer::SetIsRefreshingForMinFrameRate(
 311     bool is_refreshing_for_min_frame_rate) {
 312   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 313   PostCrossThreadTask(
 314       *io_task_runner_, FROM_HERE,
 315       CrossThreadBindOnce(&FrameDeliverer::SetIsRefreshingForMinFrameRateOnIO,
 316                           WrapRefCounted(this),
 317                           is_refreshing_for_min_frame_rate));
 318 }
 319
 320 void MediaStreamVideoTrack::FrameDeliverer::SetIsRefreshingForMinFrameRateOnIO(
 321     bool is_refreshing_for_min_frame_rate) {
 322   DCHECK(io_task_runner_->BelongsToCurrentThread());
 323   is_refreshing_for_min_frame_rate_ = is_refreshing_for_min_frame_rate;
 324 }
 325
 326 void MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnIO(
 327     scoped_refptr<media::VideoFrame> frame,
 328     std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
 329     base::TimeTicks estimated_capture_time) {
 330   DCHECK(io_task_runner_->BelongsToCurrentThread());
 331   if (!enabled_ && main_render_task_runner_ && emit_frame_drop_events_) {
 332     emit_frame_drop_events_ = false;
 333
 334     // TODO(crbug.com/964947): A weak ptr instance of MediaStreamVideoTrack is
 335     // passed to FrameDeliverer in order to avoid the re-binding the instance of
 336     // a WTF::CrossThreadFunction.
 337     PostCrossThreadTask(
 338         *main_render_task_runner_, FROM_HERE,
 339         CrossThreadBindOnce(
 340             &MediaStreamVideoTrack::OnFrameDropped, media_stream_video_track_,
 341             media::VideoCaptureFrameDropReason::
 342                 kVideoTrackFrameDelivererNotEnabledReplacingWithBlackFrame));
 343   }
 344   scoped_refptr<media::VideoFrame> video_frame;
 345   if (enabled_) {
 346     video_frame = std::move(frame);
 347   } else {
 348     // When disabled, a black video frame is passed along instead. The original
 349     // frames are dropped.
 350     video_frame = GetBlackFrame(*frame);
 351     scaled_video_frames.clear();
 352   }
 353   for (const auto& entry : callbacks_)
 354     entry.second.Run(video_frame, scaled_video_frames, estimated_capture_time);
 355
 356   // The delay on refresh timer is reset each time a frame is received so that
 357   // it will not fire for at least an additional period. This means refresh
 358   // frames will only be requested when the source has halted delivery (e.g., a
 359   // screen capturer stops sending frames because the screen is not being
 360   // updated).
 361   if (main_render_task_runner_ && is_refreshing_for_min_frame_rate_) {
 362     PostCrossThreadTask(
 363         *main_render_task_runner_, FROM_HERE,
 364         CrossThreadBindOnce(&MediaStreamVideoTrack::ResetRefreshTimer,
 365                             media_stream_video_track_));
 366   }
 367 }
 368
 369 void MediaStreamVideoTrack::FrameDeliverer::DeliverEncodedVideoFrameOnIO(
 370     scoped_refptr<EncodedVideoFrame> frame,
 371     base::TimeTicks estimated_capture_time) {
 372   DCHECK(io_task_runner_->BelongsToCurrentThread());
 373   if (!enabled_) {
 374     return;
 375   }
 376   if (await_next_key_frame_ && !frame->IsKeyFrame()) {
 377     return;
 378   }
 379   await_next_key_frame_ = false;
 380   for (const auto& entry : encoded_callbacks_.Values()) {
 381     entry.Run(frame, estimated_capture_time);
 382   }
 383 }
 384
 385 scoped_refptr<media::VideoFrame>
 386 MediaStreamVideoTrack::FrameDeliverer::GetBlackFrame(
 387     const media::VideoFrame& reference_frame) {
 388   DCHECK(io_task_runner_->BelongsToCurrentThread());
 389   if (!black_frame_.get() ||
 390       black_frame_->natural_size() != reference_frame.natural_size()) {
 391     black_frame_ =
 392         media::VideoFrame::CreateBlackFrame(reference_frame.natural_size());
 393   }
 394
 395   // Wrap |black_frame_| so we get a fresh timestamp we can modify. Frames
 396   // returned from this function may still be in use.
 397   scoped_refptr<media::VideoFrame> wrapped_black_frame =
 398       media::VideoFrame::WrapVideoFrame(black_frame_, black_frame_->format(),
 399                                         black_frame_->visible_rect(),
 400                                         black_frame_->natural_size());
 401   if (!wrapped_black_frame)
 402     return nullptr;
 403
 404   wrapped_black_frame->set_timestamp(reference_frame.timestamp());
 405   wrapped_black_frame->metadata().reference_time =
 406       reference_frame.metadata().reference_time;
 407
 408   return wrapped_black_frame;
 409 }
 410
 411 // static
 412 WebMediaStreamTrack MediaStreamVideoTrack::CreateVideoTrack(
 413     MediaStreamVideoSource* source,
 414     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 415     bool enabled) {
 416   auto* component = MakeGarbageCollected<MediaStreamComponent>(source->Owner());
 417   component->SetPlatformTrack(std::make_unique<MediaStreamVideoTrack>(
 418       source, std::move(callback), enabled));
 419   return WebMediaStreamTrack(component);
 420 }
 421
 422 // static
 423 WebMediaStreamTrack MediaStreamVideoTrack::CreateVideoTrack(
 424     MediaStreamVideoSource* source,
 425     const VideoTrackAdapterSettings& adapter_settings,
 426     const absl::optional<bool>& noise_reduction,
 427     bool is_screencast,
 428     const absl::optional<double>& min_frame_rate,
 429     const absl::optional<double>& pan,
 430     const absl::optional<double>& tilt,
 431     const absl::optional<double>& zoom,
 432     bool pan_tilt_zoom_allowed,
 433     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 434     bool enabled) {
 435   WebMediaStreamTrack track;
 436   auto* component = MakeGarbageCollected<MediaStreamComponent>(source->Owner());
 437   component->SetPlatformTrack(std::make_unique<MediaStreamVideoTrack>(
 438       source, adapter_settings, noise_reduction, is_screencast, min_frame_rate,
 439       pan, tilt, zoom, pan_tilt_zoom_allowed, std::move(callback), enabled));
 440   return WebMediaStreamTrack(component);
 441 }
 442
 443 // static
 444 MediaStreamVideoTrack* MediaStreamVideoTrack::From(
 445     const MediaStreamComponent* component) {
 446   if (!component ||
 447       component->Source()->GetType() != MediaStreamSource::kTypeVideo) {
 448     return nullptr;
 449   }
 450
 451   return static_cast<MediaStreamVideoTrack*>(component->GetPlatformTrack());
 452 }
 453
 454 MediaStreamVideoTrack::MediaStreamVideoTrack(
 455     MediaStreamVideoSource* source,
 456     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 457     bool enabled)
 458     : MediaStreamTrackPlatform(true),
 459       adapter_settings_(std::make_unique<VideoTrackAdapterSettings>(
 460           VideoTrackAdapterSettings())),
 461       is_screencast_(false),
 462       source_(source->GetWeakPtr()) {
 463   frame_deliverer_ =
 464       base::MakeRefCounted<MediaStreamVideoTrack::FrameDeliverer>(
 465           source->io_task_runner(), weak_factory_.GetWeakPtr(), enabled);
 466   source->AddTrack(
 467       this, VideoTrackAdapterSettings(),
 468       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 469           &MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnIO,
 470           frame_deliverer_)),
 471       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 472           &MediaStreamVideoTrack::FrameDeliverer::DeliverEncodedVideoFrameOnIO,
 473           frame_deliverer_)),
 474       media::BindToCurrentLoop(WTF::BindRepeating(
 475           &MediaStreamVideoTrack::SetSizeAndComputedFrameRate,
 476           weak_factory_.GetWeakPtr())),
 477       media::BindToCurrentLoop(
 478           WTF::BindRepeating(&MediaStreamVideoTrack::set_computed_source_format,
 479                              weak_factory_.GetWeakPtr())),
 480       std::move(callback));
 481 }
 482
 483 MediaStreamVideoTrack::MediaStreamVideoTrack(
 484     MediaStreamVideoSource* source,
 485     const VideoTrackAdapterSettings& adapter_settings,
 486     const absl::optional<bool>& noise_reduction,
 487     bool is_screen_cast,
 488     const absl::optional<double>& min_frame_rate,
 489     const absl::optional<double>& pan,
 490     const absl::optional<double>& tilt,
 491     const absl::optional<double>& zoom,
 492     bool pan_tilt_zoom_allowed,
 493     MediaStreamVideoSource::ConstraintsOnceCallback callback,
 494     bool enabled)
 495     : MediaStreamTrackPlatform(true),
 496       adapter_settings_(
 497           std::make_unique<VideoTrackAdapterSettings>(adapter_settings)),
 498       noise_reduction_(noise_reduction),
 499       is_screencast_(is_screen_cast),
 500       min_frame_rate_(min_frame_rate),
 501       pan_(pan),
 502       tilt_(tilt),
 503       zoom_(zoom),
 504       pan_tilt_zoom_allowed_(pan_tilt_zoom_allowed),
 505       source_(source->GetWeakPtr()) {
 506   frame_deliverer_ =
 507       base::MakeRefCounted<MediaStreamVideoTrack::FrameDeliverer>(
 508           source->io_task_runner(), weak_factory_.GetWeakPtr(), enabled);
 509   source->AddTrack(
 510       this, adapter_settings,
 511       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 512           &MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnIO,
 513           frame_deliverer_)),
 514       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 515           &MediaStreamVideoTrack::FrameDeliverer::DeliverEncodedVideoFrameOnIO,
 516           frame_deliverer_)),
 517       media::BindToCurrentLoop(WTF::BindRepeating(
 518           &MediaStreamVideoTrack::SetSizeAndComputedFrameRate,
 519           weak_factory_.GetWeakPtr())),
 520       media::BindToCurrentLoop(
 521           WTF::BindRepeating(&MediaStreamVideoTrack::set_computed_source_format,
 522                              weak_factory_.GetWeakPtr())),
 523       std::move(callback));
 524 }
 525
 526 MediaStreamVideoTrack::~MediaStreamVideoTrack() {
 527   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 528   DCHECK(sinks_.IsEmpty());
 529   DCHECK(encoded_sinks_.IsEmpty());
 530   Stop();
 531   DVLOG(3) << "~MediaStreamVideoTrack()";
 532 }
 533
 534 static void AddSinkInternal(Vector<WebMediaStreamSink*>* sinks,
 535                             WebMediaStreamSink* sink) {
 536   DCHECK(!base::Contains(*sinks, sink));
 537   sinks->push_back(sink);
 538 }
 539
 540 static void RemoveSinkInternal(Vector<WebMediaStreamSink*>* sinks,
 541                                WebMediaStreamSink* sink) {
 542   auto** it = std::find(sinks->begin(), sinks->end(), sink);
 543   DCHECK(it != sinks->end());
 544   sinks->erase(it);
 545 }
 546
 547 void MediaStreamVideoTrack::AddSink(
 548     WebMediaStreamSink* sink,
 549     const VideoCaptureDeliverFrameCB& callback,
 550     MediaStreamVideoSink::IsSecure is_secure,
 551     MediaStreamVideoSink::UsesAlpha uses_alpha) {
 552   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 553   AddSinkInternal(&sinks_, sink);
 554   frame_deliverer_->AddCallback(sink, callback);
 555   secure_tracker_.Add(sink, is_secure == MediaStreamVideoSink::IsSecure::kYes);
 556   if (uses_alpha == MediaStreamVideoSink::UsesAlpha::kDefault) {
 557     alpha_using_sinks_.insert(sink);
 558   } else if (uses_alpha == MediaStreamVideoSink::UsesAlpha::kNo) {
 559     alpha_discarding_sinks_.insert(sink);
 560   }
 561   // Request source to deliver a frame because a new sink is added.
 562   if (!source_)
 563     return;
 564   UpdateSourceHasConsumers();
 565   RequestRefreshFrame();
 566   source_->UpdateCapturingLinkSecure(this,
 567                                      secure_tracker_.is_capturing_secure());
 568   // Alpha can't be discarded if any sink uses alpha, or if the only sinks
 569   // connected are kDependsOnOtherSinks.
 570   const bool can_discard_alpha =
 571       alpha_using_sinks_.IsEmpty() && !alpha_discarding_sinks_.IsEmpty();
 572   source_->SetCanDiscardAlpha(can_discard_alpha);
 573   if (is_screencast_)
 574     StartTimerForRequestingFrames();
 575 }
 576
 577 void MediaStreamVideoTrack::AddEncodedSink(WebMediaStreamSink* sink,
 578                                            EncodedVideoFrameCB callback) {
 579   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 580   AddSinkInternal(&encoded_sinks_, sink);
 581   frame_deliverer_->AddEncodedCallback(sink, std::move(callback));
 582   if (source_)
 583     source_->UpdateNumEncodedSinks();
 584   UpdateSourceHasConsumers();
 585 }
 586
 587 void MediaStreamVideoTrack::RemoveSink(WebMediaStreamSink* sink) {
 588   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 589   RemoveSinkInternal(&sinks_, sink);
 590   alpha_using_sinks_.erase(sink);
 591   alpha_discarding_sinks_.erase(sink);
 592   frame_deliverer_->RemoveCallback(sink);
 593   secure_tracker_.Remove(sink);
 594   if (!source_)
 595     return;
 596   UpdateSourceHasConsumers();
 597   source_->UpdateCapturingLinkSecure(this,
 598                                      secure_tracker_.is_capturing_secure());
 599   const bool can_discard_alpha =
 600       sinks_.IsEmpty() ||
 601       (alpha_using_sinks_.IsEmpty() && !alpha_discarding_sinks_.IsEmpty());
 602   source_->SetCanDiscardAlpha(can_discard_alpha);
 603   // Restart the timer with existing sinks.
 604   if (is_screencast_)
 605     StartTimerForRequestingFrames();
 606 }
 607
 608 void MediaStreamVideoTrack::RemoveEncodedSink(WebMediaStreamSink* sink) {
 609   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 610   RemoveSinkInternal(&encoded_sinks_, sink);
 611   frame_deliverer_->RemoveEncodedCallback(sink);
 612   if (source_)
 613     source_->UpdateNumEncodedSinks();
 614   UpdateSourceHasConsumers();
 615 }
 616
 617 void MediaStreamVideoTrack::UpdateSourceHasConsumers() {
 618   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 619   if (!source_)
 620     return;
 621   bool has_consumers = !sinks_.IsEmpty() || !encoded_sinks_.IsEmpty();
 622   source_->UpdateHasConsumers(this, has_consumers);
 623 }
 624
 625 void MediaStreamVideoTrack::SetEnabled(bool enabled) {
 626   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 627   // If enabled, encoded sinks exist and the source supports encoded output, we
 628   // need a new keyframe from the source as we may have dropped data making the
 629   // stream undecodable.
 630   bool maybe_await_key_frame = false;
 631   if (enabled && source_ && source_->SupportsEncodedOutput() &&
 632       !encoded_sinks_.IsEmpty()) {
 633     RequestRefreshFrame();
 634     maybe_await_key_frame = true;
 635   }
 636   frame_deliverer_->SetEnabled(enabled, maybe_await_key_frame);
 637   for (auto* sink : sinks_)
 638     sink->OnEnabledChanged(enabled);
 639   for (auto* encoded_sink : encoded_sinks_)
 640     encoded_sink->OnEnabledChanged(enabled);
 641 }
 642
 643 size_t MediaStreamVideoTrack::CountSinks() const {
 644   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 645   return sinks_.size();
 646 }
 647
 648 size_t MediaStreamVideoTrack::CountEncodedSinks() const {
 649   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 650   return encoded_sinks_.size();
 651 }
 652
 653 void MediaStreamVideoTrack::SetContentHint(
 654     WebMediaStreamTrack::ContentHintType content_hint) {
 655   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 656   for (auto* sink : sinks_)
 657     sink->OnContentHintChanged(content_hint);
 658   for (auto* encoded_sink : encoded_sinks_)
 659     encoded_sink->OnContentHintChanged(content_hint);
 660 }
 661
 662 void MediaStreamVideoTrack::StopAndNotify(base::OnceClosure callback) {
 663   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 664   if (source_) {
 665     source_->RemoveTrack(this, std::move(callback));
 666     source_ = nullptr;
 667   } else if (callback) {
 668     std::move(callback).Run();
 669   }
 670   OnReadyStateChanged(WebMediaStreamSource::kReadyStateEnded);
 671   refresh_timer_.Stop();
 672 }
 673
 674 void MediaStreamVideoTrack::GetSettings(
 675     MediaStreamTrackPlatform::Settings& settings) {
 676   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 677   if (!source_)
 678     return;
 679
 680   if (width_ && height_) {
 681     settings.width = width_;
 682     settings.height = height_;
 683     settings.aspect_ratio = static_cast<double>(width_) / height_;
 684   }
 685
 686   // 0.0 means the track is using the source's frame rate.
 687   if (frame_rate_ != 0.0) {
 688     settings.frame_rate = frame_rate_;
 689   }
 690
 691   absl::optional<media::VideoCaptureFormat> format =
 692       source_->GetCurrentFormat();
 693   if (format) {
 694     if (frame_rate_ == 0.0)
 695       settings.frame_rate = format->frame_rate;
 696     settings.video_kind = GetVideoKindForFormat(*format);
 697   } else {
 698     // Format is only set for local tracks. For other tracks, use the frame rate
 699     // reported through settings callback SetSizeAndComputedFrameRate().
 700     if (computed_frame_rate_)
 701       settings.frame_rate = *computed_frame_rate_;
 702   }
 703
 704   settings.facing_mode = ToPlatformFacingMode(
 705       static_cast<mojom::blink::FacingMode>(source_->device().video_facing));
 706   settings.resize_mode = WebString::FromASCII(std::string(
 707       adapter_settings().target_size() ? WebMediaStreamTrack::kResizeModeRescale
 708                                        : WebMediaStreamTrack::kResizeModeNone));
 709   if (source_->device().display_media_info.has_value()) {
 710     const auto& info = source_->device().display_media_info.value();
 711     settings.display_surface = info->display_surface;
 712     settings.logical_surface = info->logical_surface;
 713     settings.cursor = info->cursor;
 714   }
 715 }
 716
 717 MediaStreamTrackPlatform::CaptureHandle
 718 MediaStreamVideoTrack::GetCaptureHandle() {
 719   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 720
 721   MediaStreamTrackPlatform::CaptureHandle capture_handle;
 722
 723   if (!source_) {
 724     return capture_handle;
 725   }
 726
 727   const MediaStreamDevice& device = source_->device();
 728   if (!device.display_media_info.has_value()) {
 729     return capture_handle;
 730   }
 731   const media::mojom::DisplayMediaInformationPtr& info =
 732       device.display_media_info.value();
 733
 734   if (!info->capture_handle) {
 735     return capture_handle;
 736   }
 737
 738   if (!info->capture_handle->origin.opaque()) {
 739     capture_handle.origin =
 740         String::FromUTF8(info->capture_handle->origin.Serialize());
 741   }
 742   capture_handle.handle =
 743       WebString::FromUTF16(info->capture_handle->capture_handle);
 744
 745   return capture_handle;
 746 }
 747
 748 void MediaStreamVideoTrack::OnReadyStateChanged(
 749     WebMediaStreamSource::ReadyState state) {
 750   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 751
 752   // Copy the vectors first, since sinks might DisconnectFromTrack() and
 753   // invalidate iterators.
 754
 755   Vector<WebMediaStreamSink*> sinks_copy(sinks_);
 756   for (auto* sink : sinks_copy)
 757     sink->OnReadyStateChanged(state);
 758
 759   Vector<WebMediaStreamSink*> encoded_sinks_copy(encoded_sinks_);
 760   for (auto* encoded_sink : encoded_sinks_copy)
 761     encoded_sink->OnReadyStateChanged(state);
 762 }
 763
 764 void MediaStreamVideoTrack::SetTrackAdapterSettings(
 765     const VideoTrackAdapterSettings& settings) {
 766   adapter_settings_ = std::make_unique<VideoTrackAdapterSettings>(settings);
 767 }
 768
 769 media::VideoCaptureFormat MediaStreamVideoTrack::GetComputedSourceFormat() {
 770   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 771   return computed_source_format_;
 772 }
 773
 774 void MediaStreamVideoTrack::OnFrameDropped(
 775     media::VideoCaptureFrameDropReason reason) {
 776   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 777   if (!source_)
 778     return;
 779   source_->OnFrameDropped(reason);
 780 }
 781
 782 void MediaStreamVideoTrack::SetMinimumFrameRate(double min_frame_rate) {
 783   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 784   min_frame_rate_ = min_frame_rate;
 785 }
 786
 787 void MediaStreamVideoTrack::StartTimerForRequestingFrames() {
 788   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 789
 790   // Find the maximum of all the required min frames per second in the attached
 791   // sinks.
 792   double required_min_fps = 0;
 793   for (auto* web_sink : sinks_) {
 794     auto* sink = static_cast<MediaStreamVideoSink*>(web_sink);
 795     required_min_fps =
 796         std::max(required_min_fps, sink->GetRequiredMinFramesPerSec());
 797   }
 798
 799   base::TimeDelta refresh_interval = ComputeRefreshIntervalFromBounds(
 800       base::TimeDelta::FromHz(required_min_fps), min_frame_rate_,
 801       max_frame_rate_);
 802
 803   if (refresh_interval.is_max()) {
 804     refresh_timer_.Stop();
 805     frame_deliverer_->SetIsRefreshingForMinFrameRate(false);
 806   } else {
 807     DVLOG(1) << "Starting frame refresh timer with interval "
 808              << refresh_interval.InMillisecondsF() << " ms.";
 809     refresh_timer_.Start(FROM_HERE, refresh_interval, this,
 810                          &MediaStreamVideoTrack::RequestRefreshFrame);
 811     frame_deliverer_->SetIsRefreshingForMinFrameRate(true);
 812   }
 813 }
 814
 815 void MediaStreamVideoTrack::RequestRefreshFrame() {
 816   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 817   if (source_)
 818     source_->RequestRefreshFrame();
 819 }
 820
 821 void MediaStreamVideoTrack::ResetRefreshTimer() {
 822   DCHECK_CALLED_ON_VALID_THREAD(main_render_thread_checker_);
 823   if (refresh_timer_.IsRunning())
 824     refresh_timer_.Reset();
 825 }
 826
 827 }  // namespace blink