X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=third_party%2Fblink%2Frenderer%2Fmodules%2Fmediastream%2Fvideo_track_adapter.cc;h=e8f03c2f77c4a1efd08b8c214c98f2603ff23320;hb=08cba5ce9d91575d10a867e741d43ae7fcfade13;hp=33b552da94751c24f680a10c9dd5ff3724ef6708;hpb=eaf54128d5541b5135696ac8e4f8b38eded9843b;p=platform%2Fframework%2Fweb%2Fchromium-efl.git diff --git a/third_party/blink/renderer/modules/mediastream/video_track_adapter.cc b/third_party/blink/renderer/modules/mediastream/video_track_adapter.cc index 33b552d..e8f03c2 100644 --- a/third_party/blink/renderer/modules/mediastream/video_track_adapter.cc +++ b/third_party/blink/renderer/modules/mediastream/video_track_adapter.cc @@ -11,17 +11,21 @@ #include #include -#include "base/bind.h" #include "base/containers/flat_map.h" +#include "base/functional/bind.h" #include "base/location.h" #include "base/metrics/histogram_macros.h" +#include "base/sequence_checker.h" #include "base/strings/string_number_conversions.h" -#include "base/threading/thread_task_runner_handle.h" +#include "base/task/bind_post_task.h" +#include "base/task/sequenced_task_runner.h" +#include "base/task/single_thread_task_runner.h" #include "base/trace_event/trace_event.h" #include "build/build_config.h" -#include "media/base/bind_to_current_loop.h" #include "media/base/limits.h" +#include "media/base/timestamp_constants.h" #include "media/base/video_util.h" +#include "third_party/abseil-cpp/absl/types/optional.h" #include "third_party/blink/public/common/features.h" #include "third_party/blink/public/platform/platform.h" #include "third_party/blink/renderer/modules/mediastream/video_track_adapter_settings.h" @@ -55,16 +59,18 @@ namespace { const float kFirstFrameTimeoutInFrameIntervals = 100.0f; const float kNormalFrameTimeoutInFrameIntervals = 25.0f; -// Min delta time between two frames allowed without being dropped if a max -// frame rate is specified. -constexpr base::TimeDelta kMinTimeInMsBetweenFrames = base::Milliseconds(5); +// |kMaxDeltaDeviationFactor| is used to determine |max_delta_deviation_| which +// specifies the allowed deviation from |target_delta_| before dropping a frame. +// It's set to 20% to be aligned with the previous logic in this file. +constexpr float kMaxDeltaDeviationFactor = 0.2; + // If the delta between two frames is bigger than this, we will consider it to // be invalid and reset the fps calculation. -constexpr base::TimeDelta kMaxTimeInMsBetweenFrames = base::Milliseconds(1000); +constexpr base::TimeDelta kMaxTimeBetweenFrames = base::Milliseconds(1000); -constexpr base::TimeDelta kFrameRateChangeIntervalInSeconds = base::Seconds(1); +constexpr base::TimeDelta kFrameRateChangeInterval = base::Seconds(1); const double kFrameRateChangeRate = 0.01; -constexpr base::TimeDelta kFrameRateUpdateIntervalInSeconds = base::Seconds(5); +constexpr base::TimeDelta kFrameRateUpdateInterval = base::Seconds(5); struct ComputedSettings { gfx::Size frame_size; @@ -83,6 +89,9 @@ int ClampToValidDimension(int dimension) { void ComputeFrameRate(const base::TimeDelta& frame_timestamp, double* frame_rate, base::TimeDelta* prev_frame_timestamp) { + if (frame_timestamp == media::kNoTimestamp) + return; + const double delta_ms = (frame_timestamp - *prev_frame_timestamp).InMillisecondsF(); *prev_frame_timestamp = frame_timestamp; @@ -105,8 +114,7 @@ bool MaybeUpdateFrameRate(ComputedSettings* settings) { // reported value. if (std::abs(settings->frame_rate - settings->last_updated_frame_rate) > settings->last_updated_frame_rate * kFrameRateChangeRate) { - if (now - settings->new_frame_rate_timestamp > - kFrameRateChangeIntervalInSeconds) { + if (now - settings->new_frame_rate_timestamp > kFrameRateChangeInterval) { settings->new_frame_rate_timestamp = now; settings->last_update_timestamp = now; settings->last_updated_frame_rate = settings->frame_rate; @@ -118,8 +126,7 @@ bool MaybeUpdateFrameRate(ComputedSettings* settings) { // Update frame rate if it hasn't been updated in the last // kFrameRateUpdateIntervalInSeconds seconds. - if (now - settings->last_update_timestamp > - kFrameRateUpdateIntervalInSeconds) { + if (now - settings->last_update_timestamp > kFrameRateUpdateInterval) { settings->last_update_timestamp = now; settings->last_updated_frame_rate = settings->frame_rate; return true; @@ -127,11 +134,26 @@ bool MaybeUpdateFrameRate(ComputedSettings* settings) { return false; } +VideoTrackAdapterSettings ReturnSettingsMaybeOverrideMaxFps( + const VideoTrackAdapterSettings& settings) { + VideoTrackAdapterSettings new_settings = settings; + absl::optional max_fps_override = + Platform::Current()->GetWebRtcMaxCaptureFrameRate(); + if (max_fps_override) { + DVLOG(1) << "Overriding max frame rate. Was=" + << settings.max_frame_rate().value_or(-1) + << ", Now=" << *max_fps_override; + new_settings.set_max_frame_rate(*max_fps_override); + } + return new_settings; +} + } // anonymous namespace -// VideoFrameResolutionAdapter is created on and lives on the IO-thread. It does -// the resolution adaptation and delivers frames to all registered tracks on the -// IO-thread. All method calls must be on the IO-thread. +// VideoFrameResolutionAdapter is created on and lives on the video task runner. +// It does the resolution adaptation and delivers frames to all registered +// tracks on the video task runner. All method calls must be on the video task +// runner. class VideoTrackAdapter::VideoFrameResolutionAdapter : public WTF::ThreadSafeRefCounted { public: @@ -140,14 +162,15 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter VideoCaptureNotifyFrameDroppedInternalCallback notify_frame_dropped_callback; DeliverEncodedVideoFrameInternalCallback encoded_frame_callback; - VideoCaptureCropVersionInternalCallback crop_version_callback; + VideoCaptureSubCaptureTargetVersionInternalCallback + sub_capture_target_version_callback; VideoTrackSettingsInternalCallback settings_callback; VideoTrackFormatInternalCallback format_callback; }; // Setting |max_frame_rate| to 0.0, means that no frame rate limitation // will be done. VideoFrameResolutionAdapter( - scoped_refptr render_message_loop, + scoped_refptr reader_task_runner, const VideoTrackAdapterSettings& settings, base::WeakPtr media_stream_video_source); @@ -156,17 +179,18 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter delete; // Add |frame_callback|, |encoded_frame_callback| to receive video frames on - // the IO-thread, |crop_version_callback| to receive notifications when - // a new crop version is acknowledged, and |settings_callback| to set track - // settings on the main thread. |frame_callback| will however be released - // on the main render thread. + // the video task runner, |sub_capture_target_version_callback| to receive + // notifications when a new sub-capture-target version is acknowledged, and + // |settings_callback| to set track settings on the main thread. + // |frame_callback| will however be released on the main render thread. void AddCallbacks( const MediaStreamVideoTrack* track, VideoCaptureDeliverFrameInternalCallback frame_callback, VideoCaptureNotifyFrameDroppedInternalCallback notify_frame_dropped_callback, DeliverEncodedVideoFrameInternalCallback encoded_frame_callback, - VideoCaptureCropVersionInternalCallback crop_version_callback, + VideoCaptureSubCaptureTargetVersionInternalCallback + sub_capture_target_version_callback, VideoTrackSettingsInternalCallback settings_callback, VideoTrackFormatInternalCallback format_callback); @@ -180,19 +204,21 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter // callbacks if |track| was not present in the adapter. VideoTrackCallbacks RemoveAndGetCallbacks(const MediaStreamVideoTrack* track); + // The source has provided us with a frame. void DeliverFrame( scoped_refptr frame, - std::vector> scaled_video_frames, const base::TimeTicks& estimated_capture_time, bool is_device_rotated); - - // Deliver an indication that a frame was dropped. - void DoNotifyFrameDropped(media::VideoCaptureFrameDropReason reason); + // This method is called when a frame is dropped, whether dropped by the + // source (via VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner) or + // internally (in DeliverFrame). + void OnFrameDropped(media::VideoCaptureFrameDropReason reason); void DeliverEncodedVideoFrame(scoped_refptr frame, base::TimeTicks estimated_capture_time); - void NewCropVersionOnIO(uint32_t crop_version); + void NewSubCaptureTargetVersionOnVideoTaskRunner( + uint32_t sub_capture_target_version); // Returns true if all arguments match with the output of this adapter. bool SettingsMatch(const VideoTrackAdapterSettings& settings) const; @@ -208,7 +234,6 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter void DoDeliverFrame( scoped_refptr video_frame, - std::vector> scaled_video_frames, const base::TimeTicks& estimated_capture_time); // Returns |true| if the input frame rate is higher that the requested max @@ -228,11 +253,8 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter // or frame rate have changed since last update. void MaybeUpdateTracksFormat(const media::VideoFrame& frame); - void PostFrameDroppedToMainTaskRunner( - media::VideoCaptureFrameDropReason reason); - - // Bound to the IO-thread. - THREAD_CHECKER(io_thread_checker_); + // Bound to the video task runner. + SEQUENCE_CHECKER(video_sequence_checker_); // The task runner where we will release VideoCaptureDeliverFrameCB // registered in AddCallbacks. @@ -240,10 +262,25 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter base::WeakPtr media_stream_video_source_; - VideoTrackAdapterSettings settings_; - double frame_rate_; - base::TimeDelta last_time_stamp_; - double keep_frame_counter_; + const VideoTrackAdapterSettings settings_; + + // The target timestamp delta between video frames, corresponding to the max + // fps. + const absl::optional target_delta_; + + // The maximum allowed deviation from |target_delta_| before dropping a frame. + const absl::optional max_delta_deviation_; + + // The timestamp of the last delivered video frame. + base::TimeDelta timestamp_last_delivered_frame_ = base::TimeDelta::Max(); + + // Stores the accumulated difference between |target_delta_| and the actual + // timestamp delta between frames that are delivered. Clamped to + // [-max_delta_deviation, target_delta_ / 2]. This is used to allow some + // frames to be closer than |target_delta_| in order to maintain + // |target_delta_| on average. Without it we may end up with an average fps + // that is half of max fps. + base::TimeDelta accumulated_drift_; ComputedSettings track_settings_; ComputedSettings source_format_settings_; @@ -252,30 +289,29 @@ class VideoTrackAdapter::VideoFrameResolutionAdapter }; VideoTrackAdapter::VideoFrameResolutionAdapter::VideoFrameResolutionAdapter( - scoped_refptr render_message_loop, + scoped_refptr reader_task_runner, const VideoTrackAdapterSettings& settings, base::WeakPtr media_stream_video_source) - : renderer_task_runner_(render_message_loop), + : renderer_task_runner_(reader_task_runner), media_stream_video_source_(media_stream_video_source), - settings_(settings), - frame_rate_(MediaStreamVideoSource::kDefaultFrameRate), - last_time_stamp_(base::TimeDelta::Max()), - keep_frame_counter_(0.0) { + settings_(ReturnSettingsMaybeOverrideMaxFps(settings)), + target_delta_(settings_.max_frame_rate() + ? absl::make_optional(base::Seconds( + 1.0 / settings_.max_frame_rate().value())) + : absl::nullopt), + max_delta_deviation_(target_delta_ + ? absl::make_optional(kMaxDeltaDeviationFactor * + target_delta_.value()) + : absl::nullopt) { + DVLOG(1) << __func__ << " max_framerate " + << settings.max_frame_rate().value_or(-1); DCHECK(renderer_task_runner_.get()); - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); CHECK_NE(0, settings_.max_aspect_ratio()); - - absl::optional max_fps_override = - Platform::Current()->GetWebRtcMaxCaptureFrameRate(); - if (max_fps_override) { - DVLOG(1) << "Overriding max frame rate. Was=" << settings_.max_frame_rate() - << ", Now=" << *max_fps_override; - settings_.set_max_frame_rate(*max_fps_override); - } } VideoTrackAdapter::VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); DCHECK(callbacks_.empty()); } @@ -285,10 +321,11 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallbacks( VideoCaptureNotifyFrameDroppedInternalCallback notify_frame_dropped_callback, DeliverEncodedVideoFrameInternalCallback encoded_frame_callback, - VideoCaptureCropVersionInternalCallback crop_version_callback, + VideoCaptureSubCaptureTargetVersionInternalCallback + sub_capture_target_version_callback, VideoTrackSettingsInternalCallback settings_callback, VideoTrackFormatInternalCallback format_callback) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); // The new track's settings should match the resolution adapter's current // |track_settings_| as set for existing track(s) with matching @@ -303,7 +340,7 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallbacks( std::move(frame_callback), std::move(notify_frame_dropped_callback), std::move(encoded_frame_callback), - std::move(crop_version_callback), + std::move(sub_capture_target_version_callback), std::move(settings_callback), std::move(format_callback)}; callbacks_.emplace(track, std::move(track_callbacks)); @@ -311,14 +348,14 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallbacks( void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallbacks( const MediaStreamVideoTrack* track) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); callbacks_.erase(track); } VideoTrackAdapter::VideoFrameResolutionAdapter::VideoTrackCallbacks VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveAndGetCallbacks( const MediaStreamVideoTrack* track) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); VideoTrackCallbacks track_callbacks; auto it = callbacks_.find(track); if (it == callbacks_.end()) @@ -331,14 +368,13 @@ VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveAndGetCallbacks( void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( scoped_refptr video_frame, - std::vector> scaled_video_frames, const base::TimeTicks& estimated_capture_time, bool is_device_rotated) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); if (!video_frame) { DLOG(ERROR) << "Incoming frame is not valid."; - PostFrameDroppedToMainTaskRunner( + OnFrameDropped( media::VideoCaptureFrameDropReason::kResolutionAdapterFrameIsNotValid); return; } @@ -353,7 +389,7 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( auto frame_drop_reason = media::VideoCaptureFrameDropReason::kNone; if (MaybeDropFrame(*video_frame, frame_rate, &frame_drop_reason)) { - DoNotifyFrameDropped(frame_drop_reason); + OnFrameDropped(frame_drop_reason); return; } @@ -367,16 +403,12 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( if (video_frame->HasTextures() && video_frame->storage_type() != media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) { - DoDeliverFrame(std::move(video_frame), std::move(scaled_video_frames), - estimated_capture_time); + DoDeliverFrame(std::move(video_frame), estimated_capture_time); return; } // The video frame we deliver may or may not get cropping and scaling // soft-applied. Ultimately the listener will decide whether to use the - // |delivered_video_frame| or one of the |scaled_video_frames|. When frames - // arrive to their final destination, if a scaled frame already has the - // destination dimensions there is no need to apply the soft scale calculated - // here. But that is not always possible. + // |delivered_video_frame|. scoped_refptr delivered_video_frame = video_frame; gfx::Size desired_size; @@ -391,21 +423,31 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( gfx::Rect region_in_frame = media::ComputeLetterboxRegion( video_frame->visible_rect(), desired_size); - if (video_frame->HasTextures() || video_frame->HasGpuMemoryBuffer()) { - // ComputeLetterboxRegion() produces in some cases odd dimensions due to - // internal rounding errors; |region_in_frame| is always smaller or equal - // to video_frame->visible_rect(), we can "grow it" if the dimensions are - // odd. - region_in_frame.set_width((region_in_frame.width() + 1) & ~1); - region_in_frame.set_height((region_in_frame.height() + 1) & ~1); - } + // Some consumers (for example + // ImageCaptureFrameGrabber::SingleShotFrameHandler::ConvertAndDeliverFrame) + // don't support pixel format conversions when the source format is YUV with + // UV subsampled and vsible_rect().x() being odd. The conversion ends up + // miscomputing the UV plane and ends up with a VU plane leading to a blue + // face tint. Round x() to even to avoid. See crbug.com/1307304. + region_in_frame.set_x(region_in_frame.x() & ~1); + region_in_frame.set_y(region_in_frame.y() & ~1); + + // ComputeLetterboxRegion() sometimes produces odd dimensions due to + // internal rounding errors; allow to round upwards if there's slack + // otherwise round downwards. + bool width_has_slack = + region_in_frame.right() < video_frame->visible_rect().right(); + region_in_frame.set_width((region_in_frame.width() + width_has_slack) & ~1); + bool height_has_slack = + region_in_frame.bottom() < video_frame->visible_rect().bottom(); + region_in_frame.set_height((region_in_frame.height() + height_has_slack) & + ~1); delivered_video_frame = media::VideoFrame::WrapVideoFrame( video_frame, video_frame->format(), region_in_frame, desired_size); if (!delivered_video_frame) { - PostFrameDroppedToMainTaskRunner( - media::VideoCaptureFrameDropReason:: - kResolutionAdapterWrappingFrameForCroppingFailed); + OnFrameDropped(media::VideoCaptureFrameDropReason:: + kResolutionAdapterWrappingFrameForCroppingFailed); return; } @@ -415,134 +457,112 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame( << " output visible rect " << delivered_video_frame->visible_rect().ToString(); } - DoDeliverFrame(std::move(delivered_video_frame), - std::move(scaled_video_frames), estimated_capture_time); + DoDeliverFrame(std::move(delivered_video_frame), estimated_capture_time); } void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverEncodedVideoFrame( scoped_refptr frame, base::TimeTicks estimated_capture_time) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); for (const auto& callback : callbacks_) { callback.second.encoded_frame_callback.Run(frame, estimated_capture_time); } } -void VideoTrackAdapter::VideoFrameResolutionAdapter::NewCropVersionOnIO( - uint32_t crop_version) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); +void VideoTrackAdapter::VideoFrameResolutionAdapter:: + NewSubCaptureTargetVersionOnVideoTaskRunner( + uint32_t sub_capture_target_version) { + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); for (const auto& callback : callbacks_) { - callback.second.crop_version_callback.Run(crop_version); + callback.second.sub_capture_target_version_callback.Run( + sub_capture_target_version); } } bool VideoTrackAdapter::VideoFrameResolutionAdapter::SettingsMatch( const VideoTrackAdapterSettings& settings) const { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); return settings_ == settings; } bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); return callbacks_.empty(); } void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame( scoped_refptr video_frame, - std::vector> scaled_video_frames, const base::TimeTicks& estimated_capture_time) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); if (callbacks_.empty()) { - PostFrameDroppedToMainTaskRunner( + OnFrameDropped( media::VideoCaptureFrameDropReason::kResolutionAdapterHasNoCallbacks); } for (const auto& callback : callbacks_) { MaybeUpdateTrackSettings(callback.second.settings_callback, *video_frame); - callback.second.frame_callback.Run(video_frame, scaled_video_frames, - estimated_capture_time); + callback.second.frame_callback.Run(video_frame, estimated_capture_time); } } -void VideoTrackAdapter::VideoFrameResolutionAdapter::DoNotifyFrameDropped( +void VideoTrackAdapter::VideoFrameResolutionAdapter::OnFrameDropped( media::VideoCaptureFrameDropReason reason) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); - PostFrameDroppedToMainTaskRunner(reason); - for (const auto& callback : callbacks_) - callback.second.notify_frame_dropped_callback.Run(); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); + // Notify callbacks, such as + // MediaStreamVideoTrack::FrameDeliverer::NotifyFrameDroppedOnVideoTaskRunner. + for (const auto& callback : callbacks_) { + callback.second.notify_frame_dropped_callback.Run(reason); + } } bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame( const media::VideoFrame& frame, float source_frame_rate, media::VideoCaptureFrameDropReason* reason) { - DVLOG(3) << __func__; - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); - - // Do not drop frames if max frame rate hasn't been specified. - if (settings_.max_frame_rate() == 0.0f || - (base::FeatureList::IsEnabled( - features::kMediaStreamTrackUseConfigMaxFrameRate) && - source_frame_rate > 0 && - source_frame_rate <= settings_.max_frame_rate())) { - last_time_stamp_ = frame.timestamp(); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); + + // Never drop frames if the max frame rate has not been specified. + if (!settings_.max_frame_rate().has_value()) { + timestamp_last_delivered_frame_ = frame.timestamp(); return false; } - const base::TimeDelta delta_ms = (frame.timestamp() - last_time_stamp_); + const base::TimeDelta delta = + (frame.timestamp() - timestamp_last_delivered_frame_); - // Check if the time since the last frame is completely off. - if (delta_ms.is_negative() || delta_ms > kMaxTimeInMsBetweenFrames) { - DVLOG(3) << " reset timestamps"; - // Reset |last_time_stamp_| and fps calculation. - last_time_stamp_ = frame.timestamp(); - frame_rate_ = MediaStreamVideoSource::kDefaultFrameRate; - keep_frame_counter_ = 0.0; + // Keep the frame if the time since the last frame is completely off. + if (delta.is_negative() || delta > kMaxTimeBetweenFrames) { + // Reset |timestamp_last_delivered_frame_| and |accumulated_drift|. + timestamp_last_delivered_frame_ = frame.timestamp(); + accumulated_drift_ = base::Milliseconds(0.0); return false; } - if (delta_ms < kMinTimeInMsBetweenFrames) { - // We have seen video frames being delivered from camera devices back to - // back. The simple AR filter for frame rate calculation is too short to - // handle that. https://crbug/394315 - // TODO(perkj): Can we come up with a way to fix the times stamps and the - // timing when frames are delivered so all frames can be used? - // The time stamps are generated by Chrome and not the actual device. - // Most likely the back to back problem is caused by software and not the - // actual camera. - DVLOG(3) << "Drop frame since delta time since previous frame is " - << delta_ms.InMillisecondsF() << "ms."; + DCHECK(target_delta_ && max_delta_deviation_); + if (delta < target_delta_.value() - max_delta_deviation_.value() - + accumulated_drift_) { + // Drop the frame because the input frame rate is too high. *reason = media::VideoCaptureFrameDropReason:: - kResolutionAdapterTimestampTooCloseToPrevious; + kResolutionAdapterFrameRateIsHigherThanRequested; return true; } - last_time_stamp_ = frame.timestamp(); - // Calculate the frame rate using a simple AR filter. - // Use a simple filter with 0.1 weight of the current sample. - frame_rate_ = 100 / delta_ms.InMillisecondsF() + 0.9 * frame_rate_; - DVLOG(3) << " delta_ms " << delta_ms << " frame_rate_ " << frame_rate_; - - // Prefer to not drop frames. - if (settings_.max_frame_rate() + 0.5f > frame_rate_) - return false; // Keep this frame. - - // The input frame rate is higher than requested. - // Decide if we should keep this frame or drop it. - keep_frame_counter_ += settings_.max_frame_rate() / frame_rate_; - if (keep_frame_counter_ >= 1) { - keep_frame_counter_ -= 1; - // Keep the frame. - return false; - } - DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << "."; - *reason = media::VideoCaptureFrameDropReason:: - kResolutionAdapterFrameRateIsHigherThanRequested; - return true; + + // Keep the frame and store the accumulated drift. + timestamp_last_delivered_frame_ = frame.timestamp(); + accumulated_drift_ += delta - target_delta_.value(); + DCHECK_GE(accumulated_drift_, -max_delta_deviation_.value()); + // Limit the maximum accumulated drift to half of the target delta. If we + // don't do this, it may happen that we output a series of frames too quickly + // after a period of no frames. There is no need to actively limit the minimum + // accumulated drift because that happens automatically when we drop frames + // that are too close in time. + accumulated_drift_ = std::min(accumulated_drift_, target_delta_.value() / 2); + return false; } void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTrackSettings( const VideoTrackSettingsInternalCallback& settings_callback, const media::VideoFrame& frame) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); ComputeFrameRate(frame.timestamp(), &track_settings_.frame_rate, &track_settings_.prev_frame_timestamp); if (MaybeUpdateFrameRate(&track_settings_) || @@ -554,7 +574,7 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTrackSettings( } void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTracksFormat( const media::VideoFrame& frame) { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); if (MaybeUpdateFrameRate(&source_format_settings_) || frame.natural_size() != track_settings_.frame_size) { source_format_settings_.frame_size = frame.natural_size(); @@ -567,32 +587,23 @@ void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTracksFormat( } void VideoTrackAdapter::VideoFrameResolutionAdapter::ResetFrameRate() { - DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_); for (const auto& callback : callbacks_) { callback.second.settings_callback.Run(track_settings_.frame_size, 0.0); } } -void VideoTrackAdapter::VideoFrameResolutionAdapter:: - PostFrameDroppedToMainTaskRunner( - media::VideoCaptureFrameDropReason reason) { - PostCrossThreadTask( - *renderer_task_runner_, FROM_HERE, - CrossThreadBindOnce(&MediaStreamVideoSource::OnFrameDropped, - media_stream_video_source_, reason)); -} - VideoTrackAdapter::VideoTrackAdapter( - scoped_refptr io_task_runner, + scoped_refptr video_task_runner, base::WeakPtr media_stream_video_source) - : io_task_runner_(io_task_runner), + : video_task_runner_(video_task_runner), media_stream_video_source_(media_stream_video_source), - renderer_task_runner_(base::ThreadTaskRunnerHandle::Get()), + renderer_task_runner_(base::SingleThreadTaskRunner::GetCurrentDefault()), muted_state_(false), frame_counter_(0), old_frame_counter_snapshot_(0), source_frame_rate_(0.0f) { - DCHECK(io_task_runner); + DCHECK(video_task_runner); } VideoTrackAdapter::~VideoTrackAdapter() { @@ -605,36 +616,38 @@ void VideoTrackAdapter::AddTrack( VideoCaptureDeliverFrameCB frame_callback, VideoCaptureNotifyFrameDroppedCB notify_frame_dropped_callback, EncodedVideoFrameCB encoded_frame_callback, - VideoCaptureCropVersionCB crop_version_callback, + VideoCaptureSubCaptureTargetVersionCB sub_capture_target_version_callback, VideoTrackSettingsCallback settings_callback, VideoTrackFormatCallback format_callback, const VideoTrackAdapterSettings& settings) { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, + *video_task_runner_, FROM_HERE, CrossThreadBindOnce( - &VideoTrackAdapter::AddTrackOnIO, WTF::CrossThreadUnretained(this), - WTF::CrossThreadUnretained(track), + &VideoTrackAdapter::AddTrackOnVideoTaskRunner, + WTF::CrossThreadUnretained(this), WTF::CrossThreadUnretained(track), CrossThreadBindRepeating(std::move(frame_callback)), CrossThreadBindRepeating(std::move(notify_frame_dropped_callback)), CrossThreadBindRepeating(std::move(encoded_frame_callback)), - CrossThreadBindRepeating(std::move(crop_version_callback)), + CrossThreadBindRepeating( + std::move(sub_capture_target_version_callback)), CrossThreadBindRepeating(std::move(settings_callback)), CrossThreadBindRepeating(std::move(format_callback)), settings)); } -void VideoTrackAdapter::AddTrackOnIO( +void VideoTrackAdapter::AddTrackOnVideoTaskRunner( const MediaStreamVideoTrack* track, VideoCaptureDeliverFrameInternalCallback frame_callback, VideoCaptureNotifyFrameDroppedInternalCallback notify_frame_dropped_callback, DeliverEncodedVideoFrameInternalCallback encoded_frame_callback, - VideoCaptureCropVersionInternalCallback crop_version_callback, + VideoCaptureSubCaptureTargetVersionInternalCallback + sub_capture_target_version_callback, VideoTrackSettingsInternalCallback settings_callback, VideoTrackFormatInternalCallback format_callback, const VideoTrackAdapterSettings& settings) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); scoped_refptr adapter; for (const auto& frame_adapter : adapters_) { if (frame_adapter->SettingsMatch(settings)) { @@ -648,29 +661,30 @@ void VideoTrackAdapter::AddTrackOnIO( adapters_.push_back(adapter); } - adapter->AddCallbacks( - track, std::move(frame_callback), - std::move(notify_frame_dropped_callback), - std::move(encoded_frame_callback), std::move(crop_version_callback), - std::move(settings_callback), std::move(format_callback)); + adapter->AddCallbacks(track, std::move(frame_callback), + std::move(notify_frame_dropped_callback), + std::move(encoded_frame_callback), + std::move(sub_capture_target_version_callback), + std::move(settings_callback), + std::move(format_callback)); } void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, - CrossThreadBindOnce(&VideoTrackAdapter::RemoveTrackOnIO, + *video_task_runner_, FROM_HERE, + CrossThreadBindOnce(&VideoTrackAdapter::RemoveTrackOnVideoTaskRunner, WrapRefCounted(this), CrossThreadUnretained(track))); } void VideoTrackAdapter::ReconfigureTrack( const MediaStreamVideoTrack* track, const VideoTrackAdapterSettings& settings) { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, - CrossThreadBindOnce(&VideoTrackAdapter::ReconfigureTrackOnIO, + *video_task_runner_, FROM_HERE, + CrossThreadBindOnce(&VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner, WrapRefCounted(this), CrossThreadUnretained(track), settings)); } @@ -678,33 +692,36 @@ void VideoTrackAdapter::ReconfigureTrack( void VideoTrackAdapter::StartFrameMonitoring( double source_frame_rate, const OnMutedCallback& on_muted_callback) { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); VideoTrackAdapter::OnMutedCallback bound_on_muted_callback = - media::BindToCurrentLoop(on_muted_callback); + base::BindPostTaskToCurrentDefault(on_muted_callback); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, + *video_task_runner_, FROM_HERE, CrossThreadBindOnce( - &VideoTrackAdapter::StartFrameMonitoringOnIO, WrapRefCounted(this), + &VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner, + WrapRefCounted(this), CrossThreadBindRepeating(std::move(bound_on_muted_callback)), source_frame_rate)); } void VideoTrackAdapter::StopFrameMonitoring() { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, - CrossThreadBindOnce(&VideoTrackAdapter::StopFrameMonitoringOnIO, - WrapRefCounted(this))); + *video_task_runner_, FROM_HERE, + CrossThreadBindOnce( + &VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner, + WrapRefCounted(this))); } void VideoTrackAdapter::SetSourceFrameSize(const gfx::Size& source_frame_size) { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); PostCrossThreadTask( - *io_task_runner_, FROM_HERE, - CrossThreadBindOnce(&VideoTrackAdapter::SetSourceFrameSizeOnIO, - WrapRefCounted(this), source_frame_size)); + *video_task_runner_, FROM_HERE, + CrossThreadBindOnce( + &VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner, + WrapRefCounted(this), source_frame_size)); } bool VideoTrackAdapter::CalculateDesiredSize( @@ -768,17 +785,18 @@ bool VideoTrackAdapter::CalculateDesiredSize( return true; } -void VideoTrackAdapter::StartFrameMonitoringOnIO( +void VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner( OnMutedInternalCallback on_muted_callback, double source_frame_rate) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); DCHECK(!monitoring_frame_rate_timer_); on_muted_callback_ = std::move(on_muted_callback); monitoring_frame_rate_timer_ = std::make_unique( - io_task_runner_, + video_task_runner_, ConvertToBaseRepeatingCallback(CrossThreadBindRepeating( - &VideoTrackAdapter::CheckFramesReceivedOnIO, WrapRefCounted(this)))); + &VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner, + WrapRefCounted(this)))); // If the source does not know the frame rate, set one by default. if (source_frame_rate == 0.0f) @@ -791,8 +809,8 @@ void VideoTrackAdapter::StartFrameMonitoringOnIO( base::Seconds(kFirstFrameTimeoutInFrameIntervals / source_frame_rate_)); } -void VideoTrackAdapter::StopFrameMonitoringOnIO() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); +void VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner() { + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); if (!monitoring_frame_rate_timer_) { // Already stopped. return; @@ -802,14 +820,15 @@ void VideoTrackAdapter::StopFrameMonitoringOnIO() { on_muted_callback_ = OnMutedInternalCallback(); } -void VideoTrackAdapter::SetSourceFrameSizeOnIO( +void VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner( const gfx::Size& source_frame_size) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); source_frame_size_ = source_frame_size; } -void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); +void VideoTrackAdapter::RemoveTrackOnVideoTaskRunner( + const MediaStreamVideoTrack* track) { + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); for (auto* it = adapters_.begin(); it != adapters_.end(); ++it) { (*it)->RemoveCallbacks(track); if ((*it)->IsEmpty()) { @@ -819,10 +838,10 @@ void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) { } } -void VideoTrackAdapter::ReconfigureTrackOnIO( +void VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner( const MediaStreamVideoTrack* track, const VideoTrackAdapterSettings& settings) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); VideoFrameResolutionAdapter::VideoTrackCallbacks track_callbacks; // Remove the track. @@ -839,26 +858,24 @@ void VideoTrackAdapter::ReconfigureTrackOnIO( // If the track was found, re-add it with new settings. if (track_callbacks.frame_callback) { - AddTrackOnIO(track, std::move(track_callbacks.frame_callback), - std::move(track_callbacks.notify_frame_dropped_callback), - std::move(track_callbacks.encoded_frame_callback), - std::move(track_callbacks.crop_version_callback), - std::move(track_callbacks.settings_callback), - std::move(track_callbacks.format_callback), settings); + AddTrackOnVideoTaskRunner( + track, std::move(track_callbacks.frame_callback), + std::move(track_callbacks.notify_frame_dropped_callback), + std::move(track_callbacks.encoded_frame_callback), + std::move(track_callbacks.sub_capture_target_version_callback), + std::move(track_callbacks.settings_callback), + std::move(track_callbacks.format_callback), settings); } } -void VideoTrackAdapter::DeliverFrameOnIO( +void VideoTrackAdapter::DeliverFrameOnVideoTaskRunner( scoped_refptr video_frame, - std::vector> scaled_video_frames, base::TimeTicks estimated_capture_time) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - TRACE_EVENT0("media", "VideoTrackAdapter::DeliverFrameOnIO"); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT0("media", "VideoTrackAdapter::DeliverFrameOnVideoTaskRunner"); ++frame_counter_; bool is_device_rotated = false; - // It's sufficient to look at |video_frame| since |scaled_video_frames| are - // scaled versions of the same image. // TODO(guidou): Use actual device information instead of this heuristic to // detect frames from rotated devices. https://crbug.com/722748 if (source_frame_size_ && @@ -866,38 +883,45 @@ void VideoTrackAdapter::DeliverFrameOnIO( video_frame->natural_size().height() == source_frame_size_->width()) { is_device_rotated = true; } - if (adapters_.empty()) { - PostCrossThreadTask( - *renderer_task_runner_, FROM_HERE, - CrossThreadBindOnce(&MediaStreamVideoSource::OnFrameDropped, - media_stream_video_source_, - media::VideoCaptureFrameDropReason:: - kVideoTrackAdapterHasNoResolutionAdapters)); - } for (const auto& adapter : adapters_) { - adapter->DeliverFrame(video_frame, scaled_video_frames, - estimated_capture_time, is_device_rotated); + adapter->DeliverFrame(video_frame, estimated_capture_time, + is_device_rotated); } } -void VideoTrackAdapter::DeliverEncodedVideoFrameOnIO( +void VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner( scoped_refptr frame, base::TimeTicks estimated_capture_time) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - TRACE_EVENT0("media", "VideoTrackAdapter::DeliverEncodedVideoFrameOnIO"); + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT0("media", + "VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner"); for (const auto& adapter : adapters_) adapter->DeliverEncodedVideoFrame(frame, estimated_capture_time); } -void VideoTrackAdapter::NewCropVersionOnIO(uint32_t crop_version) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - TRACE_EVENT0("media", "VideoTrackAdapter::NewCropVersionOnIO"); - for (const auto& adapter : adapters_) - adapter->NewCropVersionOnIO(crop_version); +void VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner( + media::VideoCaptureFrameDropReason reason) { + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT0("media", "VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner"); + for (const auto& adapter : adapters_) { + adapter->OnFrameDropped(reason); + } +} + +void VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner( + uint32_t sub_capture_target_version) { + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); + TRACE_EVENT0( + "media", + "VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner"); + for (const auto& adapter : adapters_) { + adapter->NewSubCaptureTargetVersionOnVideoTaskRunner( + sub_capture_target_version); + } } -void VideoTrackAdapter::CheckFramesReceivedOnIO() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); +void VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner() { + DCHECK(video_task_runner_->RunsTasksInCurrentSequence()); DVLOG_IF(1, old_frame_counter_snapshot_ == frame_counter_) << "No frames have passed, setting source as Muted.";