src/content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/debug/trace_event.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "media/base/video_util.h"
  16
  17 namespace content {
  18
  19 namespace {
  20
  21 // Amount of frame intervals to wait before considering the source as muted, for
  22 // the first frame and under normal conditions, respectively. First frame might
  23 // take longer to arrive due to source startup.
  24 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  25 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  26
  27 // Empty method used for keeping a reference to the original media::VideoFrame
  28 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  29 // The reference to |frame| is kept in the closure that calls this method.
  30 void ReleaseOriginalFrame(
  31     const scoped_refptr<media::VideoFrame>& frame) {
  32 }
  33
  34 void ResetCallbackOnMainRenderThread(
  35     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  36   // |callback| will be deleted when this exits.
  37 }
  38
  39 }  // anonymous namespace
  40
  41 // VideoFrameResolutionAdapter is created on and lives on
  42 // on the IO-thread. It does the resolution adaptation and delivers frames to
  43 // all registered tracks on the IO-thread.
  44 // All method calls must be on the IO-thread.
  45 class VideoTrackAdapter::VideoFrameResolutionAdapter
  46     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  47  public:
  48   VideoFrameResolutionAdapter(
  49       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  50       const gfx::Size& max_size,
  51       double min_aspect_ratio,
  52       double max_aspect_ratio,
  53       double max_frame_rate);
  54
  55   // Add |callback| to receive video frames on the IO-thread.
  56   // |callback| will however be released on the main render thread.
  57   void AddCallback(const MediaStreamVideoTrack* track,
  58                    const VideoCaptureDeliverFrameCB& callback);
  59
  60   // Removes |callback| associated with |track| from receiving video frames if
  61   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  62   // has not been added. The |callback| is released on the main render thread.
  63   void RemoveCallback(const MediaStreamVideoTrack* track);
  64
  65   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  66                     const media::VideoCaptureFormat& format,
  67                     const base::TimeTicks& estimated_capture_time);
  68
  69   // Returns true if all arguments match with the output of this adapter.
  70   bool ConstraintsMatch(const gfx::Size& max_size,
  71                         double min_aspect_ratio,
  72                         double max_aspect_ratio,
  73                         double max_frame_rate) const;
  74
  75   bool IsEmpty() const;
  76
  77  private:
  78   virtual ~VideoFrameResolutionAdapter();
  79   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  80
  81   virtual void DoDeliverFrame(
  82       const scoped_refptr<media::VideoFrame>& frame,
  83       const media::VideoCaptureFormat& format,
  84       const base::TimeTicks& estimated_capture_time);
  85
  86   // Returns |true| if the input frame rate is higher that the requested max
  87   // frame rate and |frame| should be dropped.
  88   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame);
  89
  90   // Bound to the IO-thread.
  91   base::ThreadChecker io_thread_checker_;
  92
  93   // The task runner where we will release VideoCaptureDeliverFrameCB
  94   // registered in AddCallback.
  95   scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
  96
  97   gfx::Size max_frame_size_;
  98   double min_aspect_ratio_;
  99   double max_aspect_ratio_;
 100
 101   double frame_rate_;
 102   base::TimeDelta last_time_stamp_;
 103   double max_frame_rate_;
 104   double keep_frame_counter_;
 105
 106   typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
 107       VideoIdCallbackPair;
 108   std::vector<VideoIdCallbackPair> callbacks_;
 109
 110   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 111 };
 112
 113 VideoTrackAdapter::
 114 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 115     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 116     const gfx::Size& max_size,
 117     double min_aspect_ratio,
 118     double max_aspect_ratio,
 119     double max_frame_rate)
 120     : renderer_task_runner_(render_message_loop),
 121       max_frame_size_(max_size),
 122       min_aspect_ratio_(min_aspect_ratio),
 123       max_aspect_ratio_(max_aspect_ratio),
 124       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 125       max_frame_rate_(max_frame_rate),
 126       keep_frame_counter_(0.0f) {
 127   DCHECK(renderer_task_runner_);
 128   DCHECK(io_thread_checker_.CalledOnValidThread());
 129   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 130   CHECK_NE(0, max_aspect_ratio_);
 131   DVLOG(3) << "VideoFrameResolutionAdapter("
 132           << "{ max_width =" << max_frame_size_.width() << "}, "
 133           << "{ max_height =" << max_frame_size_.height() << "}, "
 134           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 135           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 136           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 137 }
 138
 139 VideoTrackAdapter::
 140 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 141   DCHECK(io_thread_checker_.CalledOnValidThread());
 142   DCHECK(callbacks_.empty());
 143 }
 144
 145 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 146     const scoped_refptr<media::VideoFrame>& frame,
 147     const media::VideoCaptureFormat& format,
 148     const base::TimeTicks& estimated_capture_time) {
 149   DCHECK(io_thread_checker_.CalledOnValidThread());
 150
 151   if (MaybeDropFrame(frame))
 152     return;
 153
 154   // TODO(perkj): Allow cropping / scaling of textures once
 155   // http://crbug/362521 is fixed.
 156   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
 157     DoDeliverFrame(frame, format, estimated_capture_time);
 158     return;
 159   }
 160   scoped_refptr<media::VideoFrame> video_frame(frame);
 161   double input_ratio =
 162       static_cast<double>(frame->natural_size().width()) /
 163       frame->natural_size().height();
 164
 165   // If |frame| has larger width or height than requested, or the aspect ratio
 166   // does not match the requested, we want to create a wrapped version of this
 167   // frame with a size that fulfills the constraints.
 168   if (frame->natural_size().width() > max_frame_size_.width() ||
 169       frame->natural_size().height() > max_frame_size_.height() ||
 170       input_ratio > max_aspect_ratio_ ||
 171       input_ratio < min_aspect_ratio_) {
 172     int desired_width = std::min(max_frame_size_.width(),
 173                                  frame->natural_size().width());
 174     int desired_height = std::min(max_frame_size_.height(),
 175                                   frame->natural_size().height());
 176
 177     double resulting_ratio =
 178         static_cast<double>(desired_width) / desired_height;
 179     double requested_ratio = resulting_ratio;
 180
 181     if (requested_ratio > max_aspect_ratio_)
 182       requested_ratio = max_aspect_ratio_;
 183     else if (requested_ratio < min_aspect_ratio_)
 184       requested_ratio = min_aspect_ratio_;
 185
 186     if (resulting_ratio < requested_ratio) {
 187       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 188                                         requested_ratio);
 189       // Make sure we scale to an even height to avoid rounding errors
 190       desired_height = (desired_height + 1) & ~1;
 191     } else if (resulting_ratio > requested_ratio) {
 192       desired_width = static_cast<int>((desired_width * requested_ratio) /
 193                                        resulting_ratio);
 194       // Make sure we scale to an even width to avoid rounding errors.
 195       desired_width = (desired_width + 1) & ~1;
 196     }
 197
 198     gfx::Size desired_size(desired_width, desired_height);
 199
 200     // Get the largest centered rectangle with the same aspect ratio of
 201     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 202     // This will be the rect we need to crop the original frame to.
 203     // From this rect, the original frame can be scaled down to |desired_size|.
 204     gfx::Rect region_in_frame =
 205         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 206
 207     video_frame = media::VideoFrame::WrapVideoFrame(
 208         frame,
 209         region_in_frame,
 210         desired_size,
 211         base::Bind(&ReleaseOriginalFrame, frame));
 212
 213     DVLOG(3) << "desired size  " << desired_size.ToString()
 214              << " output natural size "
 215              << video_frame->natural_size().ToString()
 216              << " output visible rect  "
 217              << video_frame->visible_rect().ToString();
 218   }
 219   DoDeliverFrame(video_frame, format, estimated_capture_time);
 220 }
 221
 222 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 223     const scoped_refptr<media::VideoFrame>& frame) {
 224   if (max_frame_rate_ == 0.0f)
 225     return false;
 226
 227   base::TimeDelta delta = frame->timestamp() - last_time_stamp_;
 228   last_time_stamp_ = frame->timestamp();
 229   if (delta.ToInternalValue() == 0 || delta == last_time_stamp_)
 230     return false;
 231   // Calculate the moving average frame rate. Use a simple filter with 0.1
 232   // weight of the current sample.
 233   frame_rate_ = 100 / delta.InMillisecondsF() + 0.9 * frame_rate_;
 234
 235   // Prefer to not drop frames.
 236   if (max_frame_rate_ + 0.5f > frame_rate_)
 237     return false;  // Keep this frame.
 238
 239   // The input frame rate is higher than requested.
 240   // Decide if we should keep this frame or drop it.
 241   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 242   if (keep_frame_counter_ >= 1) {
 243     keep_frame_counter_ -= 1;
 244     // Keep the frame.
 245     return false;
 246   }
 247   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 248   return true;
 249 }
 250
 251 void VideoTrackAdapter::
 252 VideoFrameResolutionAdapter::DoDeliverFrame(
 253     const scoped_refptr<media::VideoFrame>& frame,
 254     const media::VideoCaptureFormat& format,
 255     const base::TimeTicks& estimated_capture_time) {
 256   DCHECK(io_thread_checker_.CalledOnValidThread());
 257   for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
 258        it != callbacks_.end(); ++it) {
 259     it->second.Run(frame, format, estimated_capture_time);
 260   }
 261 }
 262
 263 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 264     const MediaStreamVideoTrack* track,
 265     const VideoCaptureDeliverFrameCB& callback) {
 266   DCHECK(io_thread_checker_.CalledOnValidThread());
 267   callbacks_.push_back(std::make_pair(track, callback));
 268 }
 269
 270 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 271     const MediaStreamVideoTrack* track) {
 272   DCHECK(io_thread_checker_.CalledOnValidThread());
 273   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 274   for (; it != callbacks_.end(); ++it) {
 275     if (it->first == track) {
 276       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 277       // render thread since it was added on the main render thread in
 278       // VideoTrackAdapter::AddTrack.
 279       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 280           new VideoCaptureDeliverFrameCB(it->second));
 281       callbacks_.erase(it);
 282       renderer_task_runner_->PostTask(
 283           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 284                                 base::Passed(&callback)));
 285
 286       return;
 287     }
 288   }
 289 }
 290
 291 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 292     const gfx::Size& max_size,
 293     double min_aspect_ratio,
 294     double max_aspect_ratio,
 295     double max_frame_rate) const {
 296   DCHECK(io_thread_checker_.CalledOnValidThread());
 297   return max_frame_size_ == max_size &&
 298       min_aspect_ratio_ == min_aspect_ratio &&
 299       max_aspect_ratio_ == max_aspect_ratio &&
 300       max_frame_rate_ == max_frame_rate;
 301 }
 302
 303 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 304   DCHECK(io_thread_checker_.CalledOnValidThread());
 305   return callbacks_.empty();
 306 }
 307
 308 VideoTrackAdapter::VideoTrackAdapter(
 309     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
 310     : io_message_loop_(io_message_loop),
 311       renderer_task_runner_(base::MessageLoopProxy::current()),
 312       frame_counter_(0),
 313       source_frame_rate_(0.0f) {
 314   DCHECK(io_message_loop_);
 315 }
 316
 317 VideoTrackAdapter::~VideoTrackAdapter() {
 318   DCHECK(adapters_.empty());
 319   UMA_HISTOGRAM_BOOLEAN("Media.VideoTrackAdapter.FramesReceived",
 320                         frame_counter_ > 0);
 321 }
 322
 323 void VideoTrackAdapter::AddTrack(
 324     const MediaStreamVideoTrack* track,
 325     VideoCaptureDeliverFrameCB frame_callback,
 326     int max_width,
 327     int max_height,
 328     double min_aspect_ratio,
 329     double max_aspect_ratio,
 330     double max_frame_rate,
 331     double source_frame_rate,
 332     const OnMutedCallback& on_muted_state_callback) {
 333   DCHECK(thread_checker_.CalledOnValidThread());
 334   // Track monitoring should be scheduled before AddTrackOnIO() so it can find
 335   // |adapters_| empty.
 336   io_message_loop_->PostTask(
 337       FROM_HERE,
 338       base::Bind(&VideoTrackAdapter::StartTrackMonitoringOnIO,
 339                  this, on_muted_state_callback, source_frame_rate));
 340   io_message_loop_->PostTask(
 341       FROM_HERE,
 342       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
 343                  this, track, frame_callback, gfx::Size(max_width, max_height),
 344                  min_aspect_ratio, max_aspect_ratio, max_frame_rate));
 345 }
 346
 347 void VideoTrackAdapter::AddTrackOnIO(
 348     const MediaStreamVideoTrack* track,
 349     VideoCaptureDeliverFrameCB frame_callback,
 350     const gfx::Size& max_frame_size,
 351     double min_aspect_ratio,
 352     double max_aspect_ratio,
 353     double max_frame_rate) {
 354   DCHECK(io_message_loop_->BelongsToCurrentThread());
 355   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 356   for (FrameAdapters::const_iterator it = adapters_.begin();
 357        it != adapters_.end(); ++it) {
 358     if ((*it)->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 359                                 max_aspect_ratio, max_frame_rate)) {
 360       adapter = it->get();
 361       break;
 362     }
 363   }
 364   if (!adapter) {
 365     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 366                                               max_frame_size,
 367                                               min_aspect_ratio,
 368                                               max_aspect_ratio,
 369                                               max_frame_rate);
 370     adapters_.push_back(adapter);
 371   }
 372
 373   adapter->AddCallback(track, frame_callback);
 374 }
 375
 376 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 377   DCHECK(thread_checker_.CalledOnValidThread());
 378   io_message_loop_->PostTask(
 379       FROM_HERE,
 380       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 381 }
 382
 383 void VideoTrackAdapter::StartTrackMonitoringOnIO(
 384     const OnMutedCallback& on_muted_state_callback,
 385     double source_frame_rate) {
 386   DCHECK(io_message_loop_->BelongsToCurrentThread());
 387   // Only trigger monitoring for the first Track.
 388   if (!adapters_.empty())
 389     return;
 390   // If the source does not know the frame rate, set one by default.
 391   if (source_frame_rate == 0.0f)
 392     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 393   source_frame_rate_ = source_frame_rate;
 394   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 395       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 396   io_message_loop_->PostDelayedTask(FROM_HERE,
 397        base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 398                   on_muted_state_callback, frame_counter_),
 399        base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 400                                      source_frame_rate_));
 401 }
 402
 403 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 404   DCHECK(io_message_loop_->BelongsToCurrentThread());
 405   for (FrameAdapters::iterator it = adapters_.begin();
 406        it != adapters_.end(); ++it) {
 407     (*it)->RemoveCallback(track);
 408     if ((*it)->IsEmpty()) {
 409       adapters_.erase(it);
 410       break;
 411     }
 412   }
 413 }
 414
 415 void VideoTrackAdapter::DeliverFrameOnIO(
 416     const scoped_refptr<media::VideoFrame>& frame,
 417     const media::VideoCaptureFormat& format,
 418     const base::TimeTicks& estimated_capture_time) {
 419   DCHECK(io_message_loop_->BelongsToCurrentThread());
 420   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 421   ++frame_counter_;
 422   for (FrameAdapters::iterator it = adapters_.begin();
 423        it != adapters_.end(); ++it) {
 424     (*it)->DeliverFrame(frame, format, estimated_capture_time);
 425   }
 426 }
 427
 428 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 429     const OnMutedCallback& set_muted_state_callback,
 430     uint64 old_frame_counter_snapshot) {
 431   DCHECK(io_message_loop_->BelongsToCurrentThread());
 432   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 433       << "No frames have passed, setting source as Muted.";
 434   set_muted_state_callback.Run(old_frame_counter_snapshot == frame_counter_);
 435
 436   // Rearm the monitoring while there are active Tracks, i.e. as long as the
 437   // owner MediaStreamSource is active.
 438   io_message_loop_->PostDelayedTask(FROM_HERE,
 439       base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 440           set_muted_state_callback, frame_counter_),
 441       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 442                                     source_frame_rate_));
 443 }
 444
 445 }  // namespace content