#include <string>
#include <utility>
-#include "base/bind.h"
#include "base/containers/flat_map.h"
+#include "base/functional/bind.h"
#include "base/location.h"
#include "base/metrics/histogram_macros.h"
+#include "base/sequence_checker.h"
#include "base/strings/string_number_conversions.h"
-#include "base/threading/thread_task_runner_handle.h"
+#include "base/task/bind_post_task.h"
+#include "base/task/sequenced_task_runner.h"
+#include "base/task/single_thread_task_runner.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
-#include "media/base/bind_to_current_loop.h"
#include "media/base/limits.h"
+#include "media/base/timestamp_constants.h"
#include "media/base/video_util.h"
+#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/renderer/modules/mediastream/video_track_adapter_settings.h"
const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
-// Min delta time between two frames allowed without being dropped if a max
-// frame rate is specified.
-constexpr base::TimeDelta kMinTimeInMsBetweenFrames = base::Milliseconds(5);
+// |kMaxDeltaDeviationFactor| is used to determine |max_delta_deviation_| which
+// specifies the allowed deviation from |target_delta_| before dropping a frame.
+// It's set to 20% to be aligned with the previous logic in this file.
+constexpr float kMaxDeltaDeviationFactor = 0.2;
+
// If the delta between two frames is bigger than this, we will consider it to
// be invalid and reset the fps calculation.
-constexpr base::TimeDelta kMaxTimeInMsBetweenFrames = base::Milliseconds(1000);
+constexpr base::TimeDelta kMaxTimeBetweenFrames = base::Milliseconds(1000);
-constexpr base::TimeDelta kFrameRateChangeIntervalInSeconds = base::Seconds(1);
+constexpr base::TimeDelta kFrameRateChangeInterval = base::Seconds(1);
const double kFrameRateChangeRate = 0.01;
-constexpr base::TimeDelta kFrameRateUpdateIntervalInSeconds = base::Seconds(5);
+constexpr base::TimeDelta kFrameRateUpdateInterval = base::Seconds(5);
struct ComputedSettings {
gfx::Size frame_size;
void ComputeFrameRate(const base::TimeDelta& frame_timestamp,
double* frame_rate,
base::TimeDelta* prev_frame_timestamp) {
+ if (frame_timestamp == media::kNoTimestamp)
+ return;
+
const double delta_ms =
(frame_timestamp - *prev_frame_timestamp).InMillisecondsF();
*prev_frame_timestamp = frame_timestamp;
// reported value.
if (std::abs(settings->frame_rate - settings->last_updated_frame_rate) >
settings->last_updated_frame_rate * kFrameRateChangeRate) {
- if (now - settings->new_frame_rate_timestamp >
- kFrameRateChangeIntervalInSeconds) {
+ if (now - settings->new_frame_rate_timestamp > kFrameRateChangeInterval) {
settings->new_frame_rate_timestamp = now;
settings->last_update_timestamp = now;
settings->last_updated_frame_rate = settings->frame_rate;
// Update frame rate if it hasn't been updated in the last
// kFrameRateUpdateIntervalInSeconds seconds.
- if (now - settings->last_update_timestamp >
- kFrameRateUpdateIntervalInSeconds) {
+ if (now - settings->last_update_timestamp > kFrameRateUpdateInterval) {
settings->last_update_timestamp = now;
settings->last_updated_frame_rate = settings->frame_rate;
return true;
return false;
}
+VideoTrackAdapterSettings ReturnSettingsMaybeOverrideMaxFps(
+ const VideoTrackAdapterSettings& settings) {
+ VideoTrackAdapterSettings new_settings = settings;
+ absl::optional<double> max_fps_override =
+ Platform::Current()->GetWebRtcMaxCaptureFrameRate();
+ if (max_fps_override) {
+ DVLOG(1) << "Overriding max frame rate. Was="
+ << settings.max_frame_rate().value_or(-1)
+ << ", Now=" << *max_fps_override;
+ new_settings.set_max_frame_rate(*max_fps_override);
+ }
+ return new_settings;
+}
+
} // anonymous namespace
-// VideoFrameResolutionAdapter is created on and lives on the IO-thread. It does
-// the resolution adaptation and delivers frames to all registered tracks on the
-// IO-thread. All method calls must be on the IO-thread.
+// VideoFrameResolutionAdapter is created on and lives on the video task runner.
+// It does the resolution adaptation and delivers frames to all registered
+// tracks on the video task runner. All method calls must be on the video task
+// runner.
class VideoTrackAdapter::VideoFrameResolutionAdapter
: public WTF::ThreadSafeRefCounted<VideoFrameResolutionAdapter> {
public:
VideoCaptureNotifyFrameDroppedInternalCallback
notify_frame_dropped_callback;
DeliverEncodedVideoFrameInternalCallback encoded_frame_callback;
- VideoCaptureCropVersionInternalCallback crop_version_callback;
+ VideoCaptureSubCaptureTargetVersionInternalCallback
+ sub_capture_target_version_callback;
VideoTrackSettingsInternalCallback settings_callback;
VideoTrackFormatInternalCallback format_callback;
};
// Setting |max_frame_rate| to 0.0, means that no frame rate limitation
// will be done.
VideoFrameResolutionAdapter(
- scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
+ scoped_refptr<base::SingleThreadTaskRunner> reader_task_runner,
const VideoTrackAdapterSettings& settings,
base::WeakPtr<MediaStreamVideoSource> media_stream_video_source);
delete;
// Add |frame_callback|, |encoded_frame_callback| to receive video frames on
- // the IO-thread, |crop_version_callback| to receive notifications when
- // a new crop version is acknowledged, and |settings_callback| to set track
- // settings on the main thread. |frame_callback| will however be released
- // on the main render thread.
+ // the video task runner, |sub_capture_target_version_callback| to receive
+ // notifications when a new sub-capture-target version is acknowledged, and
+ // |settings_callback| to set track settings on the main thread.
+ // |frame_callback| will however be released on the main render thread.
void AddCallbacks(
const MediaStreamVideoTrack* track,
VideoCaptureDeliverFrameInternalCallback frame_callback,
VideoCaptureNotifyFrameDroppedInternalCallback
notify_frame_dropped_callback,
DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
- VideoCaptureCropVersionInternalCallback crop_version_callback,
+ VideoCaptureSubCaptureTargetVersionInternalCallback
+ sub_capture_target_version_callback,
VideoTrackSettingsInternalCallback settings_callback,
VideoTrackFormatInternalCallback format_callback);
// callbacks if |track| was not present in the adapter.
VideoTrackCallbacks RemoveAndGetCallbacks(const MediaStreamVideoTrack* track);
+ // The source has provided us with a frame.
void DeliverFrame(
scoped_refptr<media::VideoFrame> frame,
- std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
const base::TimeTicks& estimated_capture_time,
bool is_device_rotated);
-
- // Deliver an indication that a frame was dropped.
- void DoNotifyFrameDropped(media::VideoCaptureFrameDropReason reason);
+ // This method is called when a frame is dropped, whether dropped by the
+ // source (via VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner) or
+ // internally (in DeliverFrame).
+ void OnFrameDropped(media::VideoCaptureFrameDropReason reason);
void DeliverEncodedVideoFrame(scoped_refptr<EncodedVideoFrame> frame,
base::TimeTicks estimated_capture_time);
- void NewCropVersionOnIO(uint32_t crop_version);
+ void NewSubCaptureTargetVersionOnVideoTaskRunner(
+ uint32_t sub_capture_target_version);
// Returns true if all arguments match with the output of this adapter.
bool SettingsMatch(const VideoTrackAdapterSettings& settings) const;
void DoDeliverFrame(
scoped_refptr<media::VideoFrame> video_frame,
- std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
const base::TimeTicks& estimated_capture_time);
// Returns |true| if the input frame rate is higher that the requested max
// or frame rate have changed since last update.
void MaybeUpdateTracksFormat(const media::VideoFrame& frame);
- void PostFrameDroppedToMainTaskRunner(
- media::VideoCaptureFrameDropReason reason);
-
- // Bound to the IO-thread.
- THREAD_CHECKER(io_thread_checker_);
+ // Bound to the video task runner.
+ SEQUENCE_CHECKER(video_sequence_checker_);
// The task runner where we will release VideoCaptureDeliverFrameCB
// registered in AddCallbacks.
base::WeakPtr<MediaStreamVideoSource> media_stream_video_source_;
- VideoTrackAdapterSettings settings_;
- double frame_rate_;
- base::TimeDelta last_time_stamp_;
- double keep_frame_counter_;
+ const VideoTrackAdapterSettings settings_;
+
+ // The target timestamp delta between video frames, corresponding to the max
+ // fps.
+ const absl::optional<base::TimeDelta> target_delta_;
+
+ // The maximum allowed deviation from |target_delta_| before dropping a frame.
+ const absl::optional<base::TimeDelta> max_delta_deviation_;
+
+ // The timestamp of the last delivered video frame.
+ base::TimeDelta timestamp_last_delivered_frame_ = base::TimeDelta::Max();
+
+ // Stores the accumulated difference between |target_delta_| and the actual
+ // timestamp delta between frames that are delivered. Clamped to
+ // [-max_delta_deviation, target_delta_ / 2]. This is used to allow some
+ // frames to be closer than |target_delta_| in order to maintain
+ // |target_delta_| on average. Without it we may end up with an average fps
+ // that is half of max fps.
+ base::TimeDelta accumulated_drift_;
ComputedSettings track_settings_;
ComputedSettings source_format_settings_;
};
VideoTrackAdapter::VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
- scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
+ scoped_refptr<base::SingleThreadTaskRunner> reader_task_runner,
const VideoTrackAdapterSettings& settings,
base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
- : renderer_task_runner_(render_message_loop),
+ : renderer_task_runner_(reader_task_runner),
media_stream_video_source_(media_stream_video_source),
- settings_(settings),
- frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
- last_time_stamp_(base::TimeDelta::Max()),
- keep_frame_counter_(0.0) {
+ settings_(ReturnSettingsMaybeOverrideMaxFps(settings)),
+ target_delta_(settings_.max_frame_rate()
+ ? absl::make_optional(base::Seconds(
+ 1.0 / settings_.max_frame_rate().value()))
+ : absl::nullopt),
+ max_delta_deviation_(target_delta_
+ ? absl::make_optional(kMaxDeltaDeviationFactor *
+ target_delta_.value())
+ : absl::nullopt) {
+ DVLOG(1) << __func__ << " max_framerate "
+ << settings.max_frame_rate().value_or(-1);
DCHECK(renderer_task_runner_.get());
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
CHECK_NE(0, settings_.max_aspect_ratio());
-
- absl::optional<double> max_fps_override =
- Platform::Current()->GetWebRtcMaxCaptureFrameRate();
- if (max_fps_override) {
- DVLOG(1) << "Overriding max frame rate. Was=" << settings_.max_frame_rate()
- << ", Now=" << *max_fps_override;
- settings_.set_max_frame_rate(*max_fps_override);
- }
}
VideoTrackAdapter::VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
DCHECK(callbacks_.empty());
}
VideoCaptureNotifyFrameDroppedInternalCallback
notify_frame_dropped_callback,
DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
- VideoCaptureCropVersionInternalCallback crop_version_callback,
+ VideoCaptureSubCaptureTargetVersionInternalCallback
+ sub_capture_target_version_callback,
VideoTrackSettingsInternalCallback settings_callback,
VideoTrackFormatInternalCallback format_callback) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
// The new track's settings should match the resolution adapter's current
// |track_settings_| as set for existing track(s) with matching
std::move(frame_callback),
std::move(notify_frame_dropped_callback),
std::move(encoded_frame_callback),
- std::move(crop_version_callback),
+ std::move(sub_capture_target_version_callback),
std::move(settings_callback),
std::move(format_callback)};
callbacks_.emplace(track, std::move(track_callbacks));
void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallbacks(
const MediaStreamVideoTrack* track) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
callbacks_.erase(track);
}
VideoTrackAdapter::VideoFrameResolutionAdapter::VideoTrackCallbacks
VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveAndGetCallbacks(
const MediaStreamVideoTrack* track) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
VideoTrackCallbacks track_callbacks;
auto it = callbacks_.find(track);
if (it == callbacks_.end())
void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
scoped_refptr<media::VideoFrame> video_frame,
- std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
const base::TimeTicks& estimated_capture_time,
bool is_device_rotated) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
if (!video_frame) {
DLOG(ERROR) << "Incoming frame is not valid.";
- PostFrameDroppedToMainTaskRunner(
+ OnFrameDropped(
media::VideoCaptureFrameDropReason::kResolutionAdapterFrameIsNotValid);
return;
}
auto frame_drop_reason = media::VideoCaptureFrameDropReason::kNone;
if (MaybeDropFrame(*video_frame, frame_rate, &frame_drop_reason)) {
- DoNotifyFrameDropped(frame_drop_reason);
+ OnFrameDropped(frame_drop_reason);
return;
}
if (video_frame->HasTextures() &&
video_frame->storage_type() !=
media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
- DoDeliverFrame(std::move(video_frame), std::move(scaled_video_frames),
- estimated_capture_time);
+ DoDeliverFrame(std::move(video_frame), estimated_capture_time);
return;
}
// The video frame we deliver may or may not get cropping and scaling
// soft-applied. Ultimately the listener will decide whether to use the
- // |delivered_video_frame| or one of the |scaled_video_frames|. When frames
- // arrive to their final destination, if a scaled frame already has the
- // destination dimensions there is no need to apply the soft scale calculated
- // here. But that is not always possible.
+ // |delivered_video_frame|.
scoped_refptr<media::VideoFrame> delivered_video_frame = video_frame;
gfx::Size desired_size;
gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
video_frame->visible_rect(), desired_size);
- if (video_frame->HasTextures() || video_frame->HasGpuMemoryBuffer()) {
- // ComputeLetterboxRegion() produces in some cases odd dimensions due to
- // internal rounding errors; |region_in_frame| is always smaller or equal
- // to video_frame->visible_rect(), we can "grow it" if the dimensions are
- // odd.
- region_in_frame.set_width((region_in_frame.width() + 1) & ~1);
- region_in_frame.set_height((region_in_frame.height() + 1) & ~1);
- }
+ // Some consumers (for example
+ // ImageCaptureFrameGrabber::SingleShotFrameHandler::ConvertAndDeliverFrame)
+ // don't support pixel format conversions when the source format is YUV with
+ // UV subsampled and vsible_rect().x() being odd. The conversion ends up
+ // miscomputing the UV plane and ends up with a VU plane leading to a blue
+ // face tint. Round x() to even to avoid. See crbug.com/1307304.
+ region_in_frame.set_x(region_in_frame.x() & ~1);
+ region_in_frame.set_y(region_in_frame.y() & ~1);
+
+ // ComputeLetterboxRegion() sometimes produces odd dimensions due to
+ // internal rounding errors; allow to round upwards if there's slack
+ // otherwise round downwards.
+ bool width_has_slack =
+ region_in_frame.right() < video_frame->visible_rect().right();
+ region_in_frame.set_width((region_in_frame.width() + width_has_slack) & ~1);
+ bool height_has_slack =
+ region_in_frame.bottom() < video_frame->visible_rect().bottom();
+ region_in_frame.set_height((region_in_frame.height() + height_has_slack) &
+ ~1);
delivered_video_frame = media::VideoFrame::WrapVideoFrame(
video_frame, video_frame->format(), region_in_frame, desired_size);
if (!delivered_video_frame) {
- PostFrameDroppedToMainTaskRunner(
- media::VideoCaptureFrameDropReason::
- kResolutionAdapterWrappingFrameForCroppingFailed);
+ OnFrameDropped(media::VideoCaptureFrameDropReason::
+ kResolutionAdapterWrappingFrameForCroppingFailed);
return;
}
<< " output visible rect "
<< delivered_video_frame->visible_rect().ToString();
}
- DoDeliverFrame(std::move(delivered_video_frame),
- std::move(scaled_video_frames), estimated_capture_time);
+ DoDeliverFrame(std::move(delivered_video_frame), estimated_capture_time);
}
void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverEncodedVideoFrame(
scoped_refptr<EncodedVideoFrame> frame,
base::TimeTicks estimated_capture_time) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
for (const auto& callback : callbacks_) {
callback.second.encoded_frame_callback.Run(frame, estimated_capture_time);
}
}
-void VideoTrackAdapter::VideoFrameResolutionAdapter::NewCropVersionOnIO(
- uint32_t crop_version) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+void VideoTrackAdapter::VideoFrameResolutionAdapter::
+ NewSubCaptureTargetVersionOnVideoTaskRunner(
+ uint32_t sub_capture_target_version) {
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
for (const auto& callback : callbacks_) {
- callback.second.crop_version_callback.Run(crop_version);
+ callback.second.sub_capture_target_version_callback.Run(
+ sub_capture_target_version);
}
}
bool VideoTrackAdapter::VideoFrameResolutionAdapter::SettingsMatch(
const VideoTrackAdapterSettings& settings) const {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
return settings_ == settings;
}
bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
return callbacks_.empty();
}
void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
scoped_refptr<media::VideoFrame> video_frame,
- std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
const base::TimeTicks& estimated_capture_time) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
if (callbacks_.empty()) {
- PostFrameDroppedToMainTaskRunner(
+ OnFrameDropped(
media::VideoCaptureFrameDropReason::kResolutionAdapterHasNoCallbacks);
}
for (const auto& callback : callbacks_) {
MaybeUpdateTrackSettings(callback.second.settings_callback, *video_frame);
- callback.second.frame_callback.Run(video_frame, scaled_video_frames,
- estimated_capture_time);
+ callback.second.frame_callback.Run(video_frame, estimated_capture_time);
}
}
-void VideoTrackAdapter::VideoFrameResolutionAdapter::DoNotifyFrameDropped(
+void VideoTrackAdapter::VideoFrameResolutionAdapter::OnFrameDropped(
media::VideoCaptureFrameDropReason reason) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
- PostFrameDroppedToMainTaskRunner(reason);
- for (const auto& callback : callbacks_)
- callback.second.notify_frame_dropped_callback.Run();
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
+ // Notify callbacks, such as
+ // MediaStreamVideoTrack::FrameDeliverer::NotifyFrameDroppedOnVideoTaskRunner.
+ for (const auto& callback : callbacks_) {
+ callback.second.notify_frame_dropped_callback.Run(reason);
+ }
}
bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
const media::VideoFrame& frame,
float source_frame_rate,
media::VideoCaptureFrameDropReason* reason) {
- DVLOG(3) << __func__;
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
-
- // Do not drop frames if max frame rate hasn't been specified.
- if (settings_.max_frame_rate() == 0.0f ||
- (base::FeatureList::IsEnabled(
- features::kMediaStreamTrackUseConfigMaxFrameRate) &&
- source_frame_rate > 0 &&
- source_frame_rate <= settings_.max_frame_rate())) {
- last_time_stamp_ = frame.timestamp();
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
+
+ // Never drop frames if the max frame rate has not been specified.
+ if (!settings_.max_frame_rate().has_value()) {
+ timestamp_last_delivered_frame_ = frame.timestamp();
return false;
}
- const base::TimeDelta delta_ms = (frame.timestamp() - last_time_stamp_);
+ const base::TimeDelta delta =
+ (frame.timestamp() - timestamp_last_delivered_frame_);
- // Check if the time since the last frame is completely off.
- if (delta_ms.is_negative() || delta_ms > kMaxTimeInMsBetweenFrames) {
- DVLOG(3) << " reset timestamps";
- // Reset |last_time_stamp_| and fps calculation.
- last_time_stamp_ = frame.timestamp();
- frame_rate_ = MediaStreamVideoSource::kDefaultFrameRate;
- keep_frame_counter_ = 0.0;
+ // Keep the frame if the time since the last frame is completely off.
+ if (delta.is_negative() || delta > kMaxTimeBetweenFrames) {
+ // Reset |timestamp_last_delivered_frame_| and |accumulated_drift|.
+ timestamp_last_delivered_frame_ = frame.timestamp();
+ accumulated_drift_ = base::Milliseconds(0.0);
return false;
}
- if (delta_ms < kMinTimeInMsBetweenFrames) {
- // We have seen video frames being delivered from camera devices back to
- // back. The simple AR filter for frame rate calculation is too short to
- // handle that. https://crbug/394315
- // TODO(perkj): Can we come up with a way to fix the times stamps and the
- // timing when frames are delivered so all frames can be used?
- // The time stamps are generated by Chrome and not the actual device.
- // Most likely the back to back problem is caused by software and not the
- // actual camera.
- DVLOG(3) << "Drop frame since delta time since previous frame is "
- << delta_ms.InMillisecondsF() << "ms.";
+ DCHECK(target_delta_ && max_delta_deviation_);
+ if (delta < target_delta_.value() - max_delta_deviation_.value() -
+ accumulated_drift_) {
+ // Drop the frame because the input frame rate is too high.
*reason = media::VideoCaptureFrameDropReason::
- kResolutionAdapterTimestampTooCloseToPrevious;
+ kResolutionAdapterFrameRateIsHigherThanRequested;
return true;
}
- last_time_stamp_ = frame.timestamp();
- // Calculate the frame rate using a simple AR filter.
- // Use a simple filter with 0.1 weight of the current sample.
- frame_rate_ = 100 / delta_ms.InMillisecondsF() + 0.9 * frame_rate_;
- DVLOG(3) << " delta_ms " << delta_ms << " frame_rate_ " << frame_rate_;
-
- // Prefer to not drop frames.
- if (settings_.max_frame_rate() + 0.5f > frame_rate_)
- return false; // Keep this frame.
-
- // The input frame rate is higher than requested.
- // Decide if we should keep this frame or drop it.
- keep_frame_counter_ += settings_.max_frame_rate() / frame_rate_;
- if (keep_frame_counter_ >= 1) {
- keep_frame_counter_ -= 1;
- // Keep the frame.
- return false;
- }
- DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
- *reason = media::VideoCaptureFrameDropReason::
- kResolutionAdapterFrameRateIsHigherThanRequested;
- return true;
+
+ // Keep the frame and store the accumulated drift.
+ timestamp_last_delivered_frame_ = frame.timestamp();
+ accumulated_drift_ += delta - target_delta_.value();
+ DCHECK_GE(accumulated_drift_, -max_delta_deviation_.value());
+ // Limit the maximum accumulated drift to half of the target delta. If we
+ // don't do this, it may happen that we output a series of frames too quickly
+ // after a period of no frames. There is no need to actively limit the minimum
+ // accumulated drift because that happens automatically when we drop frames
+ // that are too close in time.
+ accumulated_drift_ = std::min(accumulated_drift_, target_delta_.value() / 2);
+ return false;
}
void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTrackSettings(
const VideoTrackSettingsInternalCallback& settings_callback,
const media::VideoFrame& frame) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
ComputeFrameRate(frame.timestamp(), &track_settings_.frame_rate,
&track_settings_.prev_frame_timestamp);
if (MaybeUpdateFrameRate(&track_settings_) ||
}
void VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeUpdateTracksFormat(
const media::VideoFrame& frame) {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
if (MaybeUpdateFrameRate(&source_format_settings_) ||
frame.natural_size() != track_settings_.frame_size) {
source_format_settings_.frame_size = frame.natural_size();
}
void VideoTrackAdapter::VideoFrameResolutionAdapter::ResetFrameRate() {
- DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(video_sequence_checker_);
for (const auto& callback : callbacks_) {
callback.second.settings_callback.Run(track_settings_.frame_size, 0.0);
}
}
-void VideoTrackAdapter::VideoFrameResolutionAdapter::
- PostFrameDroppedToMainTaskRunner(
- media::VideoCaptureFrameDropReason reason) {
- PostCrossThreadTask(
- *renderer_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&MediaStreamVideoSource::OnFrameDropped,
- media_stream_video_source_, reason));
-}
-
VideoTrackAdapter::VideoTrackAdapter(
- scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
+ scoped_refptr<base::SequencedTaskRunner> video_task_runner,
base::WeakPtr<MediaStreamVideoSource> media_stream_video_source)
- : io_task_runner_(io_task_runner),
+ : video_task_runner_(video_task_runner),
media_stream_video_source_(media_stream_video_source),
- renderer_task_runner_(base::ThreadTaskRunnerHandle::Get()),
+ renderer_task_runner_(base::SingleThreadTaskRunner::GetCurrentDefault()),
muted_state_(false),
frame_counter_(0),
old_frame_counter_snapshot_(0),
source_frame_rate_(0.0f) {
- DCHECK(io_task_runner);
+ DCHECK(video_task_runner);
}
VideoTrackAdapter::~VideoTrackAdapter() {
VideoCaptureDeliverFrameCB frame_callback,
VideoCaptureNotifyFrameDroppedCB notify_frame_dropped_callback,
EncodedVideoFrameCB encoded_frame_callback,
- VideoCaptureCropVersionCB crop_version_callback,
+ VideoCaptureSubCaptureTargetVersionCB sub_capture_target_version_callback,
VideoTrackSettingsCallback settings_callback,
VideoTrackFormatCallback format_callback,
const VideoTrackAdapterSettings& settings) {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
+ *video_task_runner_, FROM_HERE,
CrossThreadBindOnce(
- &VideoTrackAdapter::AddTrackOnIO, WTF::CrossThreadUnretained(this),
- WTF::CrossThreadUnretained(track),
+ &VideoTrackAdapter::AddTrackOnVideoTaskRunner,
+ WTF::CrossThreadUnretained(this), WTF::CrossThreadUnretained(track),
CrossThreadBindRepeating(std::move(frame_callback)),
CrossThreadBindRepeating(std::move(notify_frame_dropped_callback)),
CrossThreadBindRepeating(std::move(encoded_frame_callback)),
- CrossThreadBindRepeating(std::move(crop_version_callback)),
+ CrossThreadBindRepeating(
+ std::move(sub_capture_target_version_callback)),
CrossThreadBindRepeating(std::move(settings_callback)),
CrossThreadBindRepeating(std::move(format_callback)), settings));
}
-void VideoTrackAdapter::AddTrackOnIO(
+void VideoTrackAdapter::AddTrackOnVideoTaskRunner(
const MediaStreamVideoTrack* track,
VideoCaptureDeliverFrameInternalCallback frame_callback,
VideoCaptureNotifyFrameDroppedInternalCallback
notify_frame_dropped_callback,
DeliverEncodedVideoFrameInternalCallback encoded_frame_callback,
- VideoCaptureCropVersionInternalCallback crop_version_callback,
+ VideoCaptureSubCaptureTargetVersionInternalCallback
+ sub_capture_target_version_callback,
VideoTrackSettingsInternalCallback settings_callback,
VideoTrackFormatInternalCallback format_callback,
const VideoTrackAdapterSettings& settings) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
scoped_refptr<VideoFrameResolutionAdapter> adapter;
for (const auto& frame_adapter : adapters_) {
if (frame_adapter->SettingsMatch(settings)) {
adapters_.push_back(adapter);
}
- adapter->AddCallbacks(
- track, std::move(frame_callback),
- std::move(notify_frame_dropped_callback),
- std::move(encoded_frame_callback), std::move(crop_version_callback),
- std::move(settings_callback), std::move(format_callback));
+ adapter->AddCallbacks(track, std::move(frame_callback),
+ std::move(notify_frame_dropped_callback),
+ std::move(encoded_frame_callback),
+ std::move(sub_capture_target_version_callback),
+ std::move(settings_callback),
+ std::move(format_callback));
}
void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&VideoTrackAdapter::RemoveTrackOnIO,
+ *video_task_runner_, FROM_HERE,
+ CrossThreadBindOnce(&VideoTrackAdapter::RemoveTrackOnVideoTaskRunner,
WrapRefCounted(this), CrossThreadUnretained(track)));
}
void VideoTrackAdapter::ReconfigureTrack(
const MediaStreamVideoTrack* track,
const VideoTrackAdapterSettings& settings) {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&VideoTrackAdapter::ReconfigureTrackOnIO,
+ *video_task_runner_, FROM_HERE,
+ CrossThreadBindOnce(&VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner,
WrapRefCounted(this), CrossThreadUnretained(track),
settings));
}
void VideoTrackAdapter::StartFrameMonitoring(
double source_frame_rate,
const OnMutedCallback& on_muted_callback) {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
VideoTrackAdapter::OnMutedCallback bound_on_muted_callback =
- media::BindToCurrentLoop(on_muted_callback);
+ base::BindPostTaskToCurrentDefault(on_muted_callback);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
+ *video_task_runner_, FROM_HERE,
CrossThreadBindOnce(
- &VideoTrackAdapter::StartFrameMonitoringOnIO, WrapRefCounted(this),
+ &VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner,
+ WrapRefCounted(this),
CrossThreadBindRepeating(std::move(bound_on_muted_callback)),
source_frame_rate));
}
void VideoTrackAdapter::StopFrameMonitoring() {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&VideoTrackAdapter::StopFrameMonitoringOnIO,
- WrapRefCounted(this)));
+ *video_task_runner_, FROM_HERE,
+ CrossThreadBindOnce(
+ &VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner,
+ WrapRefCounted(this)));
}
void VideoTrackAdapter::SetSourceFrameSize(const gfx::Size& source_frame_size) {
- DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
PostCrossThreadTask(
- *io_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&VideoTrackAdapter::SetSourceFrameSizeOnIO,
- WrapRefCounted(this), source_frame_size));
+ *video_task_runner_, FROM_HERE,
+ CrossThreadBindOnce(
+ &VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner,
+ WrapRefCounted(this), source_frame_size));
}
bool VideoTrackAdapter::CalculateDesiredSize(
return true;
}
-void VideoTrackAdapter::StartFrameMonitoringOnIO(
+void VideoTrackAdapter::StartFrameMonitoringOnVideoTaskRunner(
OnMutedInternalCallback on_muted_callback,
double source_frame_rate) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
DCHECK(!monitoring_frame_rate_timer_);
on_muted_callback_ = std::move(on_muted_callback);
monitoring_frame_rate_timer_ = std::make_unique<LowPrecisionTimer>(
- io_task_runner_,
+ video_task_runner_,
ConvertToBaseRepeatingCallback(CrossThreadBindRepeating(
- &VideoTrackAdapter::CheckFramesReceivedOnIO, WrapRefCounted(this))));
+ &VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner,
+ WrapRefCounted(this))));
// If the source does not know the frame rate, set one by default.
if (source_frame_rate == 0.0f)
base::Seconds(kFirstFrameTimeoutInFrameIntervals / source_frame_rate_));
}
-void VideoTrackAdapter::StopFrameMonitoringOnIO() {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+void VideoTrackAdapter::StopFrameMonitoringOnVideoTaskRunner() {
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
if (!monitoring_frame_rate_timer_) {
// Already stopped.
return;
on_muted_callback_ = OnMutedInternalCallback();
}
-void VideoTrackAdapter::SetSourceFrameSizeOnIO(
+void VideoTrackAdapter::SetSourceFrameSizeOnVideoTaskRunner(
const gfx::Size& source_frame_size) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
source_frame_size_ = source_frame_size;
}
-void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+void VideoTrackAdapter::RemoveTrackOnVideoTaskRunner(
+ const MediaStreamVideoTrack* track) {
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
for (auto* it = adapters_.begin(); it != adapters_.end(); ++it) {
(*it)->RemoveCallbacks(track);
if ((*it)->IsEmpty()) {
}
}
-void VideoTrackAdapter::ReconfigureTrackOnIO(
+void VideoTrackAdapter::ReconfigureTrackOnVideoTaskRunner(
const MediaStreamVideoTrack* track,
const VideoTrackAdapterSettings& settings) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
VideoFrameResolutionAdapter::VideoTrackCallbacks track_callbacks;
// Remove the track.
// If the track was found, re-add it with new settings.
if (track_callbacks.frame_callback) {
- AddTrackOnIO(track, std::move(track_callbacks.frame_callback),
- std::move(track_callbacks.notify_frame_dropped_callback),
- std::move(track_callbacks.encoded_frame_callback),
- std::move(track_callbacks.crop_version_callback),
- std::move(track_callbacks.settings_callback),
- std::move(track_callbacks.format_callback), settings);
+ AddTrackOnVideoTaskRunner(
+ track, std::move(track_callbacks.frame_callback),
+ std::move(track_callbacks.notify_frame_dropped_callback),
+ std::move(track_callbacks.encoded_frame_callback),
+ std::move(track_callbacks.sub_capture_target_version_callback),
+ std::move(track_callbacks.settings_callback),
+ std::move(track_callbacks.format_callback), settings);
}
}
-void VideoTrackAdapter::DeliverFrameOnIO(
+void VideoTrackAdapter::DeliverFrameOnVideoTaskRunner(
scoped_refptr<media::VideoFrame> video_frame,
- std::vector<scoped_refptr<media::VideoFrame>> scaled_video_frames,
base::TimeTicks estimated_capture_time) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
- TRACE_EVENT0("media", "VideoTrackAdapter::DeliverFrameOnIO");
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
+ TRACE_EVENT0("media", "VideoTrackAdapter::DeliverFrameOnVideoTaskRunner");
++frame_counter_;
bool is_device_rotated = false;
- // It's sufficient to look at |video_frame| since |scaled_video_frames| are
- // scaled versions of the same image.
// TODO(guidou): Use actual device information instead of this heuristic to
// detect frames from rotated devices. https://crbug.com/722748
if (source_frame_size_ &&
video_frame->natural_size().height() == source_frame_size_->width()) {
is_device_rotated = true;
}
- if (adapters_.empty()) {
- PostCrossThreadTask(
- *renderer_task_runner_, FROM_HERE,
- CrossThreadBindOnce(&MediaStreamVideoSource::OnFrameDropped,
- media_stream_video_source_,
- media::VideoCaptureFrameDropReason::
- kVideoTrackAdapterHasNoResolutionAdapters));
- }
for (const auto& adapter : adapters_) {
- adapter->DeliverFrame(video_frame, scaled_video_frames,
- estimated_capture_time, is_device_rotated);
+ adapter->DeliverFrame(video_frame, estimated_capture_time,
+ is_device_rotated);
}
}
-void VideoTrackAdapter::DeliverEncodedVideoFrameOnIO(
+void VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner(
scoped_refptr<EncodedVideoFrame> frame,
base::TimeTicks estimated_capture_time) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
- TRACE_EVENT0("media", "VideoTrackAdapter::DeliverEncodedVideoFrameOnIO");
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
+ TRACE_EVENT0("media",
+ "VideoTrackAdapter::DeliverEncodedVideoFrameOnVideoTaskRunner");
for (const auto& adapter : adapters_)
adapter->DeliverEncodedVideoFrame(frame, estimated_capture_time);
}
-void VideoTrackAdapter::NewCropVersionOnIO(uint32_t crop_version) {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
- TRACE_EVENT0("media", "VideoTrackAdapter::NewCropVersionOnIO");
- for (const auto& adapter : adapters_)
- adapter->NewCropVersionOnIO(crop_version);
+void VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner(
+ media::VideoCaptureFrameDropReason reason) {
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
+ TRACE_EVENT0("media", "VideoTrackAdapter::OnFrameDroppedOnVideoTaskRunner");
+ for (const auto& adapter : adapters_) {
+ adapter->OnFrameDropped(reason);
+ }
+}
+
+void VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner(
+ uint32_t sub_capture_target_version) {
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
+ TRACE_EVENT0(
+ "media",
+ "VideoTrackAdapter::NewSubCaptureTargetVersionOnVideoTaskRunner");
+ for (const auto& adapter : adapters_) {
+ adapter->NewSubCaptureTargetVersionOnVideoTaskRunner(
+ sub_capture_target_version);
+ }
}
-void VideoTrackAdapter::CheckFramesReceivedOnIO() {
- DCHECK(io_task_runner_->BelongsToCurrentThread());
+void VideoTrackAdapter::CheckFramesReceivedOnVideoTaskRunner() {
+ DCHECK(video_task_runner_->RunsTasksInCurrentSequence());
DVLOG_IF(1, old_frame_counter_snapshot_ == frame_counter_)
<< "No frames have passed, setting source as Muted.";
projects / platform / framework / web / chromium-efl.git / blobdiff