third_party/blink/renderer/modules/mediastream/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "third_party/blink/renderer/modules/mediastream/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <cmath>
   9 #include <limits>
  10 #include <memory>
  11 #include <string>
  12 #include <utility>
  13
  14 #include "base/containers/flat_map.h"
  15 #include "base/functional/bind.h"
  16 #include "base/location.h"
  17 #include "base/metrics/histogram_macros.h"
  18 #include "base/sequence_checker.h"
  19 #include "base/strings/string_number_conversions.h"
  20 #include "base/task/bind_post_task.h"
  21 #include "base/task/sequenced_task_runner.h"
  22 #include "base/task/single_thread_task_runner.h"
  23 #include "base/trace_event/trace_event.h"
  24 #include "build/build_config.h"
  25 #include "media/base/limits.h"
  26 #include "media/base/timestamp_constants.h"
  27 #include "media/base/video_util.h"
  28 #include "third_party/abseil-cpp/absl/types/optional.h"
  29 #include "third_party/blink/public/common/features.h"
  30 #include "third_party/blink/public/platform/platform.h"
  31 #include "third_party/blink/renderer/modules/mediastream/video_track_adapter_settings.h"
  32 #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h"
  33 #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h"
  34 #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_gfx.h"
  35 #include "third_party/blink/renderer/platform/wtf/functional.h"
  36 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
  37
  38 namespace WTF {
  39
  40 // Template specializations of [1], needed to be able to pass WTF callbacks
  41 // that have VideoTrackAdapterSettings or gfx::Size parameters across threads.
  42 //
  43 // [1] third_party/blink/renderer/platform/wtf/cross_thread_copier.h.
  44 template <>
  45 struct CrossThreadCopier<blink::VideoTrackAdapterSettings>
  46     : public CrossThreadCopierPassThrough<blink::VideoTrackAdapterSettings> {
  47   STATIC_ONLY(CrossThreadCopier);
  48 };
  49
  50 }  // namespace WTF
  51
  52 namespace blink {
  53
  54 namespace {
  55
  56 // Amount of frame intervals to wait before considering the source as muted, for
  57 // the first frame and under normal conditions, respectively. First frame might
  58 // take longer to arrive due to source startup.
  59 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  60 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  61
  62 // |kMaxDeltaDeviationFactor| is used to determine |max_delta_deviation_| which
  63 // specifies the allowed deviation from |target_delta_| before dropping a frame.
  64 // It's set to 20% to be aligned with the previous logic in this file.
  65 constexpr float kMaxDeltaDeviationFactor = 0.2;
  66
  67 // If the delta between two frames is bigger than this, we will consider it to
  68 // be invalid and reset the fps calculation.
  69 constexpr base::TimeDelta kMaxTimeBetweenFrames = base::Milliseconds(1000);
  70
  71 constexpr base::TimeDelta kFrameRateChangeInterval = base::Seconds(1);
  72 const double kFrameRateChangeRate = 0.01;
  73 constexpr base::TimeDelta kFrameRateUpdateInterval = base::Seconds(5);
  74
  75 struct ComputedSettings {
  76   gfx::Size frame_size;
  77   double frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
  78   double last_updated_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
  79   base::TimeDelta prev_frame_timestamp = base::TimeDelta::Max();
  80   base::TimeTicks new_frame_rate_timestamp;
  81   base::TimeTicks last_update_timestamp;
  82 };
  83
  84 int ClampToValidDimension(int dimension) {
  85   return std::min(static_cast<int>(media::limits::kMaxDimension),
  86                   std::max(0, dimension));
  87 }
  88
  89 void ComputeFrameRate(const base::TimeDelta& frame_timestamp,
  90                       double* frame_rate,
  91                       base::TimeDelta* prev_frame_timestamp) {
  92   if (frame_timestamp == media::kNoTimestamp)
  93     return;
  94
  95   const double delta_ms =
  96       (frame_timestamp - *prev_frame_timestamp).InMillisecondsF();
  97   *prev_frame_timestamp = frame_timestamp;
  98   if (delta_ms < 0)
  99     return;
 100
 101   *frame_rate = 200 / delta_ms + 0.8 * *frame_rate;
 102 }
 103
 104 // Controls the frequency of settings updates based on frame rate changes.
 105 // Returns |true| if over the last second the computed frame rate is
 106 // consistently kFrameRateChangeRate different than the last reported value,
 107 // or if there hasn't been any update in the last
 108 // kFrameRateUpdateIntervalInSeconds seconds.
 109 bool MaybeUpdateFrameRate(ComputedSettings* settings) {
 110   base::TimeTicks now = base::TimeTicks::Now();
 111
 112   // Update frame rate if over the last second the computed frame rate has been
 113   // consistently kFrameRateChangeIntervalInSeconds different than the last
 114   // reported value.
 115   if (std::abs(settings->frame_rate - settings->last_updated_frame_rate) >
 116       settings->last_updated_frame_rate * kFrameRateChangeRate) {
 117     if (now - settings->new_frame_rate_timestamp > kFrameRateChangeInterval) {
 118       settings->new_frame_rate_timestamp = now;
 119       settings->last_update_timestamp = now;
 120       settings->last_updated_frame_rate = settings->frame_rate;
 121       return true;
 122     }
 123   } else {
 124     settings->new_frame_rate_timestamp = now;
 125   }
 126
 127   // Update frame rate if it hasn't been updated in the last
 128   // kFrameRateUpdateIntervalInSeconds seconds.
 129   if (now - settings->last_update_timestamp > kFrameRateUpdateInterval) {
 130     settings->last_update_timestamp = now;
 131     settings->last_updated_frame_rate = settings->frame_rate;
 132     return true;
 133   }
 134   return false;
 135 }
 136
 137 VideoTrackAdapterSettings ReturnSettingsMaybeOverrideMaxFps(
 138     const VideoTrackAdapterSettings& settings) {
 139   VideoTrackAdapterSettings new_settings = settings;
 140   absl::optional<double> max_fps_override =
 141       Platform::Current()->GetWebRtcMaxCaptureFrameRate();
 142   if (max_fps_override) {
 143     DVLOG(1) << "Overriding max frame rate.  Was="
 144              << settings.max_frame_rate().value_or(-1)
 145              << ", Now=" << *max_fps_override;
 146     new_settings.set_max_frame_rate(*max_fps_override);
 147   }
 148   return new_settings;
 149 }
 150
 151 }  // anonymous namespace
 152
 153 // VideoFrameResolutionAdapter is created on and lives on the video task runner.
 154 // It does the resolution adaptation and delivers frames to all registered
 155 // tracks on the video task runner. All method calls must be on the video task
 156 // runner.
 157 class VideoTrackAdapter::VideoFrameResolutionAdapter
 158     : public WTF::ThreadSafeRefCounted<VideoFrameResolutionAdapter> {
 159  public:
 160   struct VideoTrackCallbacks {
 161     VideoCaptureDeliverFrameInternalCallback frame_callback;
 162     VideoCaptureNotifyFrameDroppedInternalCallback
 163         notify_frame_dropped_callback;
 164     DeliverEncodedVideoFrameInternalCallback encoded_frame_callback;
 165     VideoCaptureSubCaptureTargetVersionInternalCallback
 166         sub_capture_target_version_callback;
 167     VideoTrackSettingsInternalCallback settings_callback;
 168     VideoTrackFormatInternalCallback format_callback;
 169   };
 170   // Setting |max_frame_rate| to 0.0, means that no frame rate limitation
 171   // will be done.
 172   VideoFrameResolutionAdapter(
 173       scoped_refptr<base::SingleThreadTaskRunner> reader_task_runner,
 174       const VideoTrackAdapterSettings& settings,
 175       base::WeakPtr<MediaStreamVideoSource> media_stream_video_source);
 176
 177   VideoFrameResolutionAdapter(const VideoFrameResolutionAdapter&) = delete;
 178   VideoFrameResolutionAdapter& operator=(const VideoFrameResolutionAdapter&) =
 179       delete;
 180
 181   // Add |frame_callback|, |encoded_frame_callback| to receive video frames on
 182   // the video task runner, |sub_capture_target_version_callback| to receive
 183   // notifications when a new sub-capture-target version is acknowledged, and
 184   // |settings_callback| to set track settings on the main thread.
 185   // |frame_callback| will however be released on the main render thread.
 186   void AddCallbacks(
 187       const MediaStreamVideoTrack* track,
 188       VideoCaptureDeliverFrameInternalCallback frame_callback,
 189       VideoCaptureNotifyFrameDroppedInternalCallback
 190           notify_frame_dropped_callback,
 191       DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
 192       VideoCaptureSubCaptureTargetVersionInternalCallback
 193           sub_capture_target_version_callback,
 194       VideoTrackSettingsInternalCallback settings_callback,
 195       VideoTrackFormatInternalCallback format_callback);
 196
 197   // Removes the callbacks associated with |track| if |track| has been added. It
 198   // is ok to call RemoveCallbacks() even if |track| has not been added.
 199   void RemoveCallbacks(const MediaStreamVideoTrack* track);
 200
 201   // Removes the callbacks associated with |track| if |track| has been added. It
 202   // is ok to call RemoveAndGetCallbacks() even if the |track| has not been
 203   // added. The function returns the callbacks if it was removed, or empty
 204   // callbacks if |track| was not present in the adapter.
 205   VideoTrackCallbacks RemoveAndGetCallbacks(const MediaStreamVideoTrack* track);
 206
 207   // The source has provided us with a frame.
 208   void DeliverFrame(
 209       scoped_refptr<media::VideoFrame> frame,
 210       const base::TimeTicks& estimated_capture_time,
 211       bool is_device_rotated);
 212   // This method is called when a frame is dropped, whether dropped by the
 213   // source (via VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner) or
 214   // internally (in DeliverFrame).
 215   void OnFrameDropped(media::VideoCaptureFrameDropReason reason);
 216
 217   void DeliverEncodedVideoFrame(scoped_refptr<EncodedVideoFrame> frame,
 218                                 base::TimeTicks estimated_capture_time);
 219
 220   void NewSubCaptureTargetVersionOnVideoTaskRunner(
 221       uint32_t sub_capture_target_version);
 222
 223   // Returns true if all arguments match with the output of this adapter.
 224   bool SettingsMatch(const VideoTrackAdapterSettings& settings) const;
 225
 226   bool IsEmpty() const;
 227
 228   // Sets frame rate to 0.0 if frame monitor has detected muted state.
 229   void ResetFrameRate();
 230
 231  private:
 232   virtual ~VideoFrameResolutionAdapter();
 233   friend class WTF::ThreadSafeRefCounted<VideoFrameResolutionAdapter>;
 234
 235   void DoDeliverFrame(
 236       scoped_refptr<media::VideoFrame> video_frame,
 237       const base::TimeTicks& estimated_capture_time);
 238
 239   // Returns |true| if the input frame rate is higher that the requested max
 240   // frame rate and |frame| should be dropped. If it returns true, |reason| is
 241   // assigned to indicate the particular reason for the decision.
 242   bool MaybeDropFrame(const media::VideoFrame& frame,
 243                       float source_frame_rate,
 244                       media::VideoCaptureFrameDropReason* reason);
 245
 246   // Updates track settings if either frame width, height or frame rate have
 247   // changed since last update.
 248   void MaybeUpdateTrackSettings(
 249       const VideoTrackSettingsInternalCallback& settings_callback,
 250       const media::VideoFrame& frame);
 251
 252   // Updates computed source format for all tracks if either frame width, height
 253   // or frame rate have changed since last update.
 254   void MaybeUpdateTracksFormat(const media::VideoFrame& frame);
 255
 256   // Bound to the video task runner.
 257   SEQUENCE_CHECKER(video_sequence_checker_);
 258
 259   // The task runner where we will release VideoCaptureDeliverFrameCB
 260   // registered in AddCallbacks.
 261   const scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 262
 263   base::WeakPtr<MediaStreamVideoSource> media_stream_video_source_;
 264
 265   const VideoTrackAdapterSettings settings_;
 266
 267   // The target timestamp delta between video frames, corresponding to the max
 268   // fps.
 269   const absl::optional<base::TimeDelta> target_delta_;
 270
 271   // The maximum allowed deviation from |target_delta_| before dropping a frame.
 272   const absl::optional<base::TimeDelta> max_delta_deviation_;
 273
 274   // The timestamp of the last delivered video frame.
 275   base::TimeDelta timestamp_last_delivered_frame_ = base::TimeDelta::Max();
 276
 277   // Stores the accumulated difference between |target_delta_| and the actual
 278   // timestamp delta between frames that are delivered. Clamped to
 279   // [-max_delta_deviation, target_delta_ / 2]. This is used to allow some
 280   // frames to be closer than |target_delta_| in order to maintain
 281   // |target_delta_| on average. Without it we may end up with an average fps
 282   // that is half of max fps.
 283   base::TimeDelta accumulated_drift_;
 284
 285   ComputedSettings track_settings_;
 286   ComputedSettings source_format_settings_;
 287
 288   base::flat_map<const MediaStreamVideoTrack*, VideoTrackCallbacks> callbacks_;
 289 };
 290
 291 VideoTrackAdapter::VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 292     scoped_refptr<base::SingleThreadTaskRunner> reader_task_runner,
 293     const VideoTrackAdapterSettings& settings,
 294     base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
 295     : renderer_task_runner_(reader_task_runner),
 296       media_stream_video_source_(media_stream_video_source),
 297       settings_(ReturnSettingsMaybeOverrideMaxFps(settings)),
 298       target_delta_(settings_.max_frame_rate()
 299                         ? absl::make_optional(base::Seconds(
 300                               1.0 / settings_.max_frame_rate().value()))
 301                         : absl::nullopt),
 302       max_delta_deviation_(target_delta_
 303                                ? absl::make_optional(kMaxDeltaDeviationFactor *
 304                                                      target_delta_.value())
 305                                : absl::nullopt) {
 306   DVLOG(1) << __func__ << " max_framerate "
 307            << settings.max_frame_rate().value_or(-1);
 308   DCHECK(renderer_task_runner_.get());
 309   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 310   CHECK_NE(0, settings_.max_aspect_ratio());
 311 }
 312
 313 VideoTrackAdapter::VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 314   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 315   DCHECK(callbacks_.empty());
 316 }
 317
 318 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallbacks(
 319     const MediaStreamVideoTrack* track,
 320     VideoCaptureDeliverFrameInternalCallback frame_callback,
 321     VideoCaptureNotifyFrameDroppedInternalCallback
 322         notify_frame_dropped_callback,
 323     DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
 324     VideoCaptureSubCaptureTargetVersionInternalCallback
 325         sub_capture_target_version_callback,
 326     VideoTrackSettingsInternalCallback settings_callback,
 327     VideoTrackFormatInternalCallback format_callback) {
 328   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 329
 330   // The new track's settings should match the resolution adapter's current
 331   // |track_settings_| as set for existing track(s) with matching
 332   // VideoTrackAdapterSettings.
 333   if (!callbacks_.empty() && track_settings_.frame_size.width() > 0 &&
 334       track_settings_.frame_size.height() > 0) {
 335     settings_callback.Run(track_settings_.frame_size,
 336                           track_settings_.frame_rate);
 337   }
 338
 339   VideoTrackCallbacks track_callbacks = {
 340       std::move(frame_callback),
 341       std::move(notify_frame_dropped_callback),
 342       std::move(encoded_frame_callback),
 343       std::move(sub_capture_target_version_callback),
 344       std::move(settings_callback),
 345       std::move(format_callback)};
 346   callbacks_.emplace(track, std::move(track_callbacks));
 347 }
 348
 349 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallbacks(
 350     const MediaStreamVideoTrack* track) {
 351   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 352   callbacks_.erase(track);
 353 }
 354
 355 VideoTrackAdapter::VideoFrameResolutionAdapter::VideoTrackCallbacks
 356 VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveAndGetCallbacks(
 357     const MediaStreamVideoTrack* track) {
 358   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 359   VideoTrackCallbacks track_callbacks;
 360   auto it = callbacks_.find(track);
 361   if (it == callbacks_.end())
 362     return track_callbacks;
 363
 364   track_callbacks = std::move(it->second);
 365   callbacks_.erase(it);
 366   return track_callbacks;
 367 }
 368
 369 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 370     scoped_refptr<media::VideoFrame> video_frame,
 371     const base::TimeTicks& estimated_capture_time,
 372     bool is_device_rotated) {
 373   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 374
 375   if (!video_frame) {
 376     DLOG(ERROR) << "Incoming frame is not valid.";
 377     OnFrameDropped(
 378         media::VideoCaptureFrameDropReason::kResolutionAdapterFrameIsNotValid);
 379     return;
 380   }
 381
 382   ComputeFrameRate(video_frame->timestamp(),
 383                    &source_format_settings_.frame_rate,
 384                    &source_format_settings_.prev_frame_timestamp);
 385   MaybeUpdateTracksFormat(*video_frame);
 386
 387   double frame_rate = video_frame->metadata().frame_rate.value_or(
 388       MediaStreamVideoSource::kUnknownFrameRate);
 389
 390   auto frame_drop_reason = media::VideoCaptureFrameDropReason::kNone;
 391   if (MaybeDropFrame(*video_frame, frame_rate, &frame_drop_reason)) {
 392     OnFrameDropped(frame_drop_reason);
 393     return;
 394   }
 395
 396   // If the frame is a texture not backed up by GPU memory we don't apply
 397   // cropping/scaling and deliver the frame as-is, leaving it up to the
 398   // destination to rescale it. Otherwise, cropping and scaling is soft-applied
 399   // before delivery for efficiency.
 400   //
 401   // TODO(crbug.com/362521): Allow cropping/scaling of non-GPU memory backed
 402   // textures.
 403   if (video_frame->HasTextures() &&
 404       video_frame->storage_type() !=
 405           media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
 406     DoDeliverFrame(std::move(video_frame), estimated_capture_time);
 407     return;
 408   }
 409   // The video frame we deliver may or may not get cropping and scaling
 410   // soft-applied. Ultimately the listener will decide whether to use the
 411   // |delivered_video_frame|.
 412   scoped_refptr<media::VideoFrame> delivered_video_frame = video_frame;
 413
 414   gfx::Size desired_size;
 415   CalculateDesiredSize(is_device_rotated, video_frame->natural_size(),
 416                        settings_, &desired_size);
 417   if (desired_size != video_frame->natural_size()) {
 418     // Get the largest centered rectangle with the same aspect ratio of
 419     // |desired_size| that fits entirely inside of
 420     // |video_frame->visible_rect()|. This will be the rect we need to crop the
 421     // original frame to. From this rect, the original frame can be scaled down
 422     // to |desired_size|.
 423     gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
 424         video_frame->visible_rect(), desired_size);
 425
 426     // Some consumers (for example
 427     // ImageCaptureFrameGrabber::SingleShotFrameHandler::ConvertAndDeliverFrame)
 428     // don't support pixel format conversions when the source format is YUV with
 429     // UV subsampled and vsible_rect().x() being odd. The conversion ends up
 430     // miscomputing the UV plane and ends up with a VU plane leading to a blue
 431     // face tint. Round x() to even to avoid. See crbug.com/1307304.
 432     region_in_frame.set_x(region_in_frame.x() & ~1);
 433     region_in_frame.set_y(region_in_frame.y() & ~1);
 434
 435     // ComputeLetterboxRegion() sometimes produces odd dimensions due to
 436     // internal rounding errors; allow to round upwards if there's slack
 437     // otherwise round downwards.
 438     bool width_has_slack =
 439         region_in_frame.right() < video_frame->visible_rect().right();
 440     region_in_frame.set_width((region_in_frame.width() + width_has_slack) & ~1);
 441     bool height_has_slack =
 442         region_in_frame.bottom() < video_frame->visible_rect().bottom();
 443     region_in_frame.set_height((region_in_frame.height() + height_has_slack) &
 444                                ~1);
 445
 446     delivered_video_frame = media::VideoFrame::WrapVideoFrame(
 447         video_frame, video_frame->format(), region_in_frame, desired_size);
 448     if (!delivered_video_frame) {
 449       OnFrameDropped(media::VideoCaptureFrameDropReason::
 450                          kResolutionAdapterWrappingFrameForCroppingFailed);
 451       return;
 452     }
 453
 454     DVLOG(3) << "desired size  " << desired_size.ToString()
 455              << " output natural size "
 456              << delivered_video_frame->natural_size().ToString()
 457              << " output visible rect  "
 458              << delivered_video_frame->visible_rect().ToString();
 459   }
 460   DoDeliverFrame(std::move(delivered_video_frame), estimated_capture_time);
 461 }
 462
 463 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverEncodedVideoFrame(
 464     scoped_refptr<EncodedVideoFrame> frame,
 465     base::TimeTicks estimated_capture_time) {
 466   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 467   for (const auto& callback : callbacks_) {
 468     callback.second.encoded_frame_callback.Run(frame, estimated_capture_time);
 469   }
 470 }
 471
 472 void VideoTrackAdapter::VideoFrameResolutionAdapter::
 473     NewSubCaptureTargetVersionOnVideoTaskRunner(
 474         uint32_t sub_capture_target_version) {
 475   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 476   for (const auto& callback : callbacks_) {
 477     callback.second.sub_capture_target_version_callback.Run(
 478         sub_capture_target_version);
 479   }
 480 }
 481
 482 bool VideoTrackAdapter::VideoFrameResolutionAdapter::SettingsMatch(
 483     const VideoTrackAdapterSettings& settings) const {
 484   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 485   return settings_ == settings;
 486 }
 487
 488 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 489   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 490   return callbacks_.empty();
 491 }
 492
 493 void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
 494     scoped_refptr<media::VideoFrame> video_frame,
 495     const base::TimeTicks& estimated_capture_time) {
 496   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 497   if (callbacks_.empty()) {
 498     OnFrameDropped(
 499         media::VideoCaptureFrameDropReason::kResolutionAdapterHasNoCallbacks);
 500   }
 501   for (const auto& callback : callbacks_) {
 502     MaybeUpdateTrackSettings(callback.second.settings_callback, *video_frame);
 503     callback.second.frame_callback.Run(video_frame, estimated_capture_time);
 504   }
 505 }
 506
 507 void VideoTrackAdapter::VideoFrameResolutionAdapter::OnFrameDropped(
 508     media::VideoCaptureFrameDropReason reason) {
 509   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 510   // Notify callbacks, such as
 511   // MediaStreamVideoTrack::FrameDeliverer::NotifyFrameDroppedOnVideoTaskRunner.
 512   for (const auto& callback : callbacks_) {
 513     callback.second.notify_frame_dropped_callback.Run(reason);
 514   }
 515 }
 516
 517 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 518     const media::VideoFrame& frame,
 519     float source_frame_rate,
 520     media::VideoCaptureFrameDropReason* reason) {
 521   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 522
 523   // Never drop frames if the max frame rate has not been specified.
 524   if (!settings_.max_frame_rate().has_value()) {
 525     timestamp_last_delivered_frame_ = frame.timestamp();
 526     return false;
 527   }
 528
 529   const base::TimeDelta delta =
 530       (frame.timestamp() - timestamp_last_delivered_frame_);
 531
 532   // Keep the frame if the time since the last frame is completely off.
 533   if (delta.is_negative() || delta > kMaxTimeBetweenFrames) {
 534     // Reset |timestamp_last_delivered_frame_| and |accumulated_drift|.
 535     timestamp_last_delivered_frame_ = frame.timestamp();
 536     accumulated_drift_ = base::Milliseconds(0.0);
 537     return false;
 538   }
 539
 540   DCHECK(target_delta_ && max_delta_deviation_);
 541   if (delta < target_delta_.value() - max_delta_deviation_.value() -
 542                   accumulated_drift_) {
 543     // Drop the frame because the input frame rate is too high.
 544     *reason = media::VideoCaptureFrameDropReason::
 545         kResolutionAdapterFrameRateIsHigherThanRequested;
 546     return true;
 547   }
 548
 549   // Keep the frame and store the accumulated drift.
 550   timestamp_last_delivered_frame_ = frame.timestamp();
 551   accumulated_drift_ += delta - target_delta_.value();
 552   DCHECK_GE(accumulated_drift_, -max_delta_deviation_.value());
 553   // Limit the maximum accumulated drift to half of the target delta. If we
 554   // don't do this, it may happen that we output a series of frames too quickly
 555   // after a period of no frames. There is no need to actively limit the minimum
 556   // accumulated drift because that happens automatically when we drop frames
 557   // that are too close in time.
 558   accumulated_drift_ = std::min(accumulated_drift_, target_delta_.value() / 2);
 559   return false;
 560 }
 561
 562 void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTrackSettings(
 563     const VideoTrackSettingsInternalCallback& settings_callback,
 564     const media::VideoFrame& frame) {
 565   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 566   ComputeFrameRate(frame.timestamp(), &track_settings_.frame_rate,
 567                    &track_settings_.prev_frame_timestamp);
 568   if (MaybeUpdateFrameRate(&track_settings_) ||
 569       frame.natural_size() != track_settings_.frame_size) {
 570     track_settings_.frame_size = frame.natural_size();
 571     settings_callback.Run(track_settings_.frame_size,
 572                           track_settings_.frame_rate);
 573   }
 574 }
 575 void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTracksFormat(
 576     const media::VideoFrame& frame) {
 577   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 578   if (MaybeUpdateFrameRate(&source_format_settings_) ||
 579       frame.natural_size() != track_settings_.frame_size) {
 580     source_format_settings_.frame_size = frame.natural_size();
 581     media::VideoCaptureFormat source_format;
 582     source_format.frame_size = source_format_settings_.frame_size;
 583     source_format.frame_rate = source_format_settings_.frame_rate;
 584     for (const auto& callback : callbacks_)
 585       callback.second.format_callback.Run(source_format);
 586   }
 587 }
 588
 589 void VideoTrackAdapter::VideoFrameResolutionAdapter::ResetFrameRate() {
 590   DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
 591   for (const auto& callback : callbacks_) {
 592     callback.second.settings_callback.Run(track_settings_.frame_size, 0.0);
 593   }
 594 }
 595
 596 VideoTrackAdapter::VideoTrackAdapter(
 597     scoped_refptr<base::SequencedTaskRunner> video_task_runner,
 598     base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
 599     : video_task_runner_(video_task_runner),
 600       media_stream_video_source_(media_stream_video_source),
 601       renderer_task_runner_(base::SingleThreadTaskRunner::GetCurrentDefault()),
 602       muted_state_(false),
 603       frame_counter_(0),
 604       old_frame_counter_snapshot_(0),
 605       source_frame_rate_(0.0f) {
 606   DCHECK(video_task_runner);
 607 }
 608
 609 VideoTrackAdapter::~VideoTrackAdapter() {
 610   DCHECK(adapters_.empty());
 611   DCHECK(!monitoring_frame_rate_timer_);
 612 }
 613
 614 void VideoTrackAdapter::AddTrack(
 615     const MediaStreamVideoTrack* track,
 616     VideoCaptureDeliverFrameCB frame_callback,
 617     VideoCaptureNotifyFrameDroppedCB notify_frame_dropped_callback,
 618     EncodedVideoFrameCB encoded_frame_callback,
 619     VideoCaptureSubCaptureTargetVersionCB sub_capture_target_version_callback,
 620     VideoTrackSettingsCallback settings_callback,
 621     VideoTrackFormatCallback format_callback,
 622     const VideoTrackAdapterSettings& settings) {
 623   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 624
 625   PostCrossThreadTask(
 626       *video_task_runner_, FROM_HERE,
 627       CrossThreadBindOnce(
 628           &VideoTrackAdapter::AddTrackOnVideoTaskRunner,
 629           WTF::CrossThreadUnretained(this), WTF::CrossThreadUnretained(track),
 630           CrossThreadBindRepeating(std::move(frame_callback)),
 631           CrossThreadBindRepeating(std::move(notify_frame_dropped_callback)),
 632           CrossThreadBindRepeating(std::move(encoded_frame_callback)),
 633           CrossThreadBindRepeating(
 634               std::move(sub_capture_target_version_callback)),
 635           CrossThreadBindRepeating(std::move(settings_callback)),
 636           CrossThreadBindRepeating(std::move(format_callback)), settings));
 637 }
 638
 639 void VideoTrackAdapter::AddTrackOnVideoTaskRunner(
 640     const MediaStreamVideoTrack* track,
 641     VideoCaptureDeliverFrameInternalCallback frame_callback,
 642     VideoCaptureNotifyFrameDroppedInternalCallback
 643         notify_frame_dropped_callback,
 644     DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
 645     VideoCaptureSubCaptureTargetVersionInternalCallback
 646         sub_capture_target_version_callback,
 647     VideoTrackSettingsInternalCallback settings_callback,
 648     VideoTrackFormatInternalCallback format_callback,
 649     const VideoTrackAdapterSettings& settings) {
 650   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 651   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 652   for (const auto& frame_adapter : adapters_) {
 653     if (frame_adapter->SettingsMatch(settings)) {
 654       adapter = frame_adapter.get();
 655       break;
 656     }
 657   }
 658   if (!adapter.get()) {
 659     adapter = base::MakeRefCounted<VideoFrameResolutionAdapter>(
 660         renderer_task_runner_, settings, media_stream_video_source_);
 661     adapters_.push_back(adapter);
 662   }
 663
 664   adapter->AddCallbacks(track, std::move(frame_callback),
 665                         std::move(notify_frame_dropped_callback),
 666                         std::move(encoded_frame_callback),
 667                         std::move(sub_capture_target_version_callback),
 668                         std::move(settings_callback),
 669                         std::move(format_callback));
 670 }
 671
 672 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 673   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 674   PostCrossThreadTask(
 675       *video_task_runner_, FROM_HERE,
 676       CrossThreadBindOnce(&VideoTrackAdapter::RemoveTrackOnVideoTaskRunner,
 677                           WrapRefCounted(this), CrossThreadUnretained(track)));
 678 }
 679
 680 void VideoTrackAdapter::ReconfigureTrack(
 681     const MediaStreamVideoTrack* track,
 682     const VideoTrackAdapterSettings& settings) {
 683   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 684
 685   PostCrossThreadTask(
 686       *video_task_runner_, FROM_HERE,
 687       CrossThreadBindOnce(&VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner,
 688                           WrapRefCounted(this), CrossThreadUnretained(track),
 689                           settings));
 690 }
 691
 692 void VideoTrackAdapter::StartFrameMonitoring(
 693     double source_frame_rate,
 694     const OnMutedCallback& on_muted_callback) {
 695   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 696
 697   VideoTrackAdapter::OnMutedCallback bound_on_muted_callback =
 698       base::BindPostTaskToCurrentDefault(on_muted_callback);
 699
 700   PostCrossThreadTask(
 701       *video_task_runner_, FROM_HERE,
 702       CrossThreadBindOnce(
 703           &VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner,
 704           WrapRefCounted(this),
 705           CrossThreadBindRepeating(std::move(bound_on_muted_callback)),
 706           source_frame_rate));
 707 }
 708
 709 void VideoTrackAdapter::StopFrameMonitoring() {
 710   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 711   PostCrossThreadTask(
 712       *video_task_runner_, FROM_HERE,
 713       CrossThreadBindOnce(
 714           &VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner,
 715           WrapRefCounted(this)));
 716 }
 717
 718 void VideoTrackAdapter::SetSourceFrameSize(const gfx::Size& source_frame_size) {
 719   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 720   PostCrossThreadTask(
 721       *video_task_runner_, FROM_HERE,
 722       CrossThreadBindOnce(
 723           &VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner,
 724           WrapRefCounted(this), source_frame_size));
 725 }
 726
 727 bool VideoTrackAdapter::CalculateDesiredSize(
 728     bool is_rotated,
 729     const gfx::Size& original_input_size,
 730     const VideoTrackAdapterSettings& settings,
 731     gfx::Size* desired_size) {
 732   // Perform all the rescaling computations as if the device was never rotated.
 733   int width =
 734       is_rotated ? original_input_size.height() : original_input_size.width();
 735   int height =
 736       is_rotated ? original_input_size.width() : original_input_size.height();
 737   DCHECK_GE(width, 0);
 738   DCHECK_GE(height, 0);
 739
 740   // Rescale only if a target size was provided in |settings|.
 741   if (settings.target_size()) {
 742     // Adjust the size of the frame to the maximum allowed size.
 743     width =
 744         ClampToValidDimension(std::min(width, settings.target_size()->width()));
 745     height = ClampToValidDimension(
 746         std::min(height, settings.target_size()->height()));
 747
 748     // If the area of the frame is zero, ignore aspect-ratio correction.
 749     if (width * height > 0) {
 750       double ratio = static_cast<double>(width) / height;
 751       DCHECK(std::isfinite(ratio));
 752       if (ratio > settings.max_aspect_ratio() ||
 753           ratio < settings.min_aspect_ratio()) {
 754         // Make sure |min_aspect_ratio| <= |desired_ratio| <=
 755         // |max_aspect_ratio|.
 756         double desired_ratio =
 757             std::max(std::min(ratio, settings.max_aspect_ratio()),
 758                      settings.min_aspect_ratio());
 759         DCHECK(std::isfinite(desired_ratio));
 760         DCHECK_NE(desired_ratio, 0.0);
 761
 762         if (ratio < desired_ratio) {
 763           double desired_height_fp = (height * ratio) / desired_ratio;
 764           DCHECK(std::isfinite(desired_height_fp));
 765           height = static_cast<int>(desired_height_fp);
 766           // Make sure we scale to an even height to avoid rounding errors
 767           height = (height + 1) & ~1;
 768         } else if (ratio > desired_ratio) {
 769           double desired_width_fp = (width * desired_ratio) / ratio;
 770           DCHECK(std::isfinite(desired_width_fp));
 771           width = static_cast<int>(desired_width_fp);
 772           // Make sure we scale to an even width to avoid rounding errors.
 773           width = (width + 1) & ~1;
 774         }
 775       }
 776     }
 777   } else if (width > media::limits::kMaxDimension ||
 778              height > media::limits::kMaxDimension) {
 779     return false;
 780   }
 781
 782   // Output back taking device rotation into account.
 783   *desired_size =
 784       is_rotated ? gfx::Size(height, width) : gfx::Size(width, height);
 785   return true;
 786 }
 787
 788 void VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner(
 789     OnMutedInternalCallback on_muted_callback,
 790     double source_frame_rate) {
 791   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 792   DCHECK(!monitoring_frame_rate_timer_);
 793
 794   on_muted_callback_ = std::move(on_muted_callback);
 795   monitoring_frame_rate_timer_ = std::make_unique<LowPrecisionTimer>(
 796       video_task_runner_,
 797       ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
 798           &VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner,
 799           WrapRefCounted(this))));
 800
 801   // If the source does not know the frame rate, set one by default.
 802   if (source_frame_rate == 0.0f)
 803     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 804   source_frame_rate_ = source_frame_rate;
 805   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 806            << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 807   old_frame_counter_snapshot_ = frame_counter_;
 808   monitoring_frame_rate_timer_->StartOneShot(
 809       base::Seconds(kFirstFrameTimeoutInFrameIntervals / source_frame_rate_));
 810 }
 811
 812 void VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner() {
 813   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 814   if (!monitoring_frame_rate_timer_) {
 815     // Already stopped.
 816     return;
 817   }
 818   monitoring_frame_rate_timer_->Shutdown();
 819   monitoring_frame_rate_timer_.reset();
 820   on_muted_callback_ = OnMutedInternalCallback();
 821 }
 822
 823 void VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner(
 824     const gfx::Size& source_frame_size) {
 825   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 826   source_frame_size_ = source_frame_size;
 827 }
 828
 829 void VideoTrackAdapter::RemoveTrackOnVideoTaskRunner(
 830     const MediaStreamVideoTrack* track) {
 831   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 832   for (auto* it = adapters_.begin(); it != adapters_.end(); ++it) {
 833     (*it)->RemoveCallbacks(track);
 834     if ((*it)->IsEmpty()) {
 835       adapters_.erase(it);
 836       break;
 837     }
 838   }
 839 }
 840
 841 void VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner(
 842     const MediaStreamVideoTrack* track,
 843     const VideoTrackAdapterSettings& settings) {
 844   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 845
 846   VideoFrameResolutionAdapter::VideoTrackCallbacks track_callbacks;
 847   // Remove the track.
 848   for (auto* it = adapters_.begin(); it != adapters_.end(); ++it) {
 849     track_callbacks = (*it)->RemoveAndGetCallbacks(track);
 850     if (!track_callbacks.frame_callback)
 851       continue;
 852     if ((*it)->IsEmpty()) {
 853       DCHECK(track_callbacks.frame_callback);
 854       adapters_.erase(it);
 855     }
 856     break;
 857   }
 858
 859   // If the track was found, re-add it with new settings.
 860   if (track_callbacks.frame_callback) {
 861     AddTrackOnVideoTaskRunner(
 862         track, std::move(track_callbacks.frame_callback),
 863         std::move(track_callbacks.notify_frame_dropped_callback),
 864         std::move(track_callbacks.encoded_frame_callback),
 865         std::move(track_callbacks.sub_capture_target_version_callback),
 866         std::move(track_callbacks.settings_callback),
 867         std::move(track_callbacks.format_callback), settings);
 868   }
 869 }
 870
 871 void VideoTrackAdapter::DeliverFrameOnVideoTaskRunner(
 872     scoped_refptr<media::VideoFrame> video_frame,
 873     base::TimeTicks estimated_capture_time) {
 874   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 875   TRACE_EVENT0("media", "VideoTrackAdapter::DeliverFrameOnVideoTaskRunner");
 876   ++frame_counter_;
 877
 878   bool is_device_rotated = false;
 879   // TODO(guidou): Use actual device information instead of this heuristic to
 880   // detect frames from rotated devices. https://crbug.com/722748
 881   if (source_frame_size_ &&
 882       video_frame->natural_size().width() == source_frame_size_->height() &&
 883       video_frame->natural_size().height() == source_frame_size_->width()) {
 884     is_device_rotated = true;
 885   }
 886   for (const auto& adapter : adapters_) {
 887     adapter->DeliverFrame(video_frame, estimated_capture_time,
 888                           is_device_rotated);
 889   }
 890 }
 891
 892 void VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner(
 893     scoped_refptr<EncodedVideoFrame> frame,
 894     base::TimeTicks estimated_capture_time) {
 895   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 896   TRACE_EVENT0("media",
 897                "VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner");
 898   for (const auto& adapter : adapters_)
 899     adapter->DeliverEncodedVideoFrame(frame, estimated_capture_time);
 900 }
 901
 902 void VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner(
 903     media::VideoCaptureFrameDropReason reason) {
 904   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 905   TRACE_EVENT0("media", "VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner");
 906   for (const auto& adapter : adapters_) {
 907     adapter->OnFrameDropped(reason);
 908   }
 909 }
 910
 911 void VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner(
 912     uint32_t sub_capture_target_version) {
 913   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 914   TRACE_EVENT0(
 915       "media",
 916       "VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner");
 917   for (const auto& adapter : adapters_) {
 918     adapter->NewSubCaptureTargetVersionOnVideoTaskRunner(
 919         sub_capture_target_version);
 920   }
 921 }
 922
 923 void VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner() {
 924   DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
 925
 926   DVLOG_IF(1, old_frame_counter_snapshot_ == frame_counter_)
 927       << "No frames have passed, setting source as Muted.";
 928   bool muted_state = old_frame_counter_snapshot_ == frame_counter_;
 929   if (muted_state_ != muted_state) {
 930     on_muted_callback_.Run(muted_state);
 931     muted_state_ = muted_state;
 932     if (muted_state_) {
 933       for (const auto& adapter : adapters_)
 934         adapter->ResetFrameRate();
 935     }
 936   }
 937
 938   old_frame_counter_snapshot_ = frame_counter_;
 939   monitoring_frame_rate_timer_->StartOneShot(
 940       base::Seconds(kNormalFrameTimeoutInFrameIntervals / source_frame_rate_));
 941 }
 942
 943 }  // namespace blink