src/content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/debug/trace_event.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "media/base/video_util.h"
  16
  17 namespace content {
  18
  19 namespace {
  20
  21 // Amount of frame intervals to wait before considering the source as muted, for
  22 // the first frame and under normal conditions, respectively. First frame might
  23 // take longer to arrive due to source startup.
  24 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  25 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  26
  27 // Min delta time between two frames allowed without being dropped if a max
  28 // frame rate is specified.
  29 const int kMinTimeInMsBetweenFrames = 5;
  30
  31 // Empty method used for keeping a reference to the original media::VideoFrame
  32 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  33 // The reference to |frame| is kept in the closure that calls this method.
  34 void ReleaseOriginalFrame(
  35     const scoped_refptr<media::VideoFrame>& frame) {
  36 }
  37
  38 void ResetCallbackOnMainRenderThread(
  39     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  40   // |callback| will be deleted when this exits.
  41 }
  42
  43 }  // anonymous namespace
  44
  45 // VideoFrameResolutionAdapter is created on and lives on
  46 // on the IO-thread. It does the resolution adaptation and delivers frames to
  47 // all registered tracks on the IO-thread.
  48 // All method calls must be on the IO-thread.
  49 class VideoTrackAdapter::VideoFrameResolutionAdapter
  50     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  51  public:
  52   VideoFrameResolutionAdapter(
  53       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  54       const gfx::Size& max_size,
  55       double min_aspect_ratio,
  56       double max_aspect_ratio,
  57       double max_frame_rate);
  58
  59   // Add |callback| to receive video frames on the IO-thread.
  60   // |callback| will however be released on the main render thread.
  61   void AddCallback(const MediaStreamVideoTrack* track,
  62                    const VideoCaptureDeliverFrameCB& callback);
  63
  64   // Removes |callback| associated with |track| from receiving video frames if
  65   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  66   // has not been added. The |callback| is released on the main render thread.
  67   void RemoveCallback(const MediaStreamVideoTrack* track);
  68
  69   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  70                     const media::VideoCaptureFormat& format,
  71                     const base::TimeTicks& estimated_capture_time);
  72
  73   // Returns true if all arguments match with the output of this adapter.
  74   bool ConstraintsMatch(const gfx::Size& max_size,
  75                         double min_aspect_ratio,
  76                         double max_aspect_ratio,
  77                         double max_frame_rate) const;
  78
  79   bool IsEmpty() const;
  80
  81  private:
  82   virtual ~VideoFrameResolutionAdapter();
  83   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  84
  85   virtual void DoDeliverFrame(
  86       const scoped_refptr<media::VideoFrame>& frame,
  87       const media::VideoCaptureFormat& format,
  88       const base::TimeTicks& estimated_capture_time);
  89
  90   // Returns |true| if the input frame rate is higher that the requested max
  91   // frame rate and |frame| should be dropped.
  92   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame,
  93                       float source_frame_rate);
  94
  95   // Bound to the IO-thread.
  96   base::ThreadChecker io_thread_checker_;
  97
  98   // The task runner where we will release VideoCaptureDeliverFrameCB
  99   // registered in AddCallback.
 100   scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 101
 102   gfx::Size max_frame_size_;
 103   double min_aspect_ratio_;
 104   double max_aspect_ratio_;
 105
 106   double frame_rate_;
 107   base::TimeDelta last_time_stamp_;
 108   double max_frame_rate_;
 109   double keep_frame_counter_;
 110
 111   typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
 112       VideoIdCallbackPair;
 113   std::vector<VideoIdCallbackPair> callbacks_;
 114
 115   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 116 };
 117
 118 VideoTrackAdapter::
 119 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 120     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 121     const gfx::Size& max_size,
 122     double min_aspect_ratio,
 123     double max_aspect_ratio,
 124     double max_frame_rate)
 125     : renderer_task_runner_(render_message_loop),
 126       max_frame_size_(max_size),
 127       min_aspect_ratio_(min_aspect_ratio),
 128       max_aspect_ratio_(max_aspect_ratio),
 129       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 130       max_frame_rate_(max_frame_rate),
 131       keep_frame_counter_(0.0f) {
 132   DCHECK(renderer_task_runner_);
 133   DCHECK(io_thread_checker_.CalledOnValidThread());
 134   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 135   CHECK_NE(0, max_aspect_ratio_);
 136   DVLOG(3) << "VideoFrameResolutionAdapter("
 137           << "{ max_width =" << max_frame_size_.width() << "}, "
 138           << "{ max_height =" << max_frame_size_.height() << "}, "
 139           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 140           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 141           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 142 }
 143
 144 VideoTrackAdapter::
 145 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 146   DCHECK(io_thread_checker_.CalledOnValidThread());
 147   DCHECK(callbacks_.empty());
 148 }
 149
 150 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 151     const scoped_refptr<media::VideoFrame>& frame,
 152     const media::VideoCaptureFormat& format,
 153     const base::TimeTicks& estimated_capture_time) {
 154   DCHECK(io_thread_checker_.CalledOnValidThread());
 155
 156   if (MaybeDropFrame(frame, format.frame_rate))
 157     return;
 158
 159   // TODO(perkj): Allow cropping / scaling of textures once
 160   // http://crbug/362521 is fixed.
 161   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
 162     DoDeliverFrame(frame, format, estimated_capture_time);
 163     return;
 164   }
 165   scoped_refptr<media::VideoFrame> video_frame(frame);
 166   double input_ratio =
 167       static_cast<double>(frame->natural_size().width()) /
 168       frame->natural_size().height();
 169
 170   // If |frame| has larger width or height than requested, or the aspect ratio
 171   // does not match the requested, we want to create a wrapped version of this
 172   // frame with a size that fulfills the constraints.
 173   if (frame->natural_size().width() > max_frame_size_.width() ||
 174       frame->natural_size().height() > max_frame_size_.height() ||
 175       input_ratio > max_aspect_ratio_ ||
 176       input_ratio < min_aspect_ratio_) {
 177     int desired_width = std::min(max_frame_size_.width(),
 178                                  frame->natural_size().width());
 179     int desired_height = std::min(max_frame_size_.height(),
 180                                   frame->natural_size().height());
 181
 182     double resulting_ratio =
 183         static_cast<double>(desired_width) / desired_height;
 184     double requested_ratio = resulting_ratio;
 185
 186     if (requested_ratio > max_aspect_ratio_)
 187       requested_ratio = max_aspect_ratio_;
 188     else if (requested_ratio < min_aspect_ratio_)
 189       requested_ratio = min_aspect_ratio_;
 190
 191     if (resulting_ratio < requested_ratio) {
 192       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 193                                         requested_ratio);
 194       // Make sure we scale to an even height to avoid rounding errors
 195       desired_height = (desired_height + 1) & ~1;
 196     } else if (resulting_ratio > requested_ratio) {
 197       desired_width = static_cast<int>((desired_width * requested_ratio) /
 198                                        resulting_ratio);
 199       // Make sure we scale to an even width to avoid rounding errors.
 200       desired_width = (desired_width + 1) & ~1;
 201     }
 202
 203     gfx::Size desired_size(desired_width, desired_height);
 204
 205     // Get the largest centered rectangle with the same aspect ratio of
 206     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 207     // This will be the rect we need to crop the original frame to.
 208     // From this rect, the original frame can be scaled down to |desired_size|.
 209     gfx::Rect region_in_frame =
 210         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 211
 212     video_frame = media::VideoFrame::WrapVideoFrame(
 213         frame,
 214         region_in_frame,
 215         desired_size,
 216         base::Bind(&ReleaseOriginalFrame, frame));
 217
 218     DVLOG(3) << "desired size  " << desired_size.ToString()
 219              << " output natural size "
 220              << video_frame->natural_size().ToString()
 221              << " output visible rect  "
 222              << video_frame->visible_rect().ToString();
 223   }
 224   DoDeliverFrame(video_frame, format, estimated_capture_time);
 225 }
 226
 227 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 228     const scoped_refptr<media::VideoFrame>& frame,
 229     float source_frame_rate) {
 230   DCHECK(io_thread_checker_.CalledOnValidThread());
 231
 232   // Do not drop frames if max frame rate hasn't been specified or the source
 233   // frame rate is known and is lower than max.
 234   if (max_frame_rate_ == 0.0f ||
 235       (source_frame_rate > 0 &&
 236           source_frame_rate <= max_frame_rate_))
 237     return false;
 238
 239   base::TimeDelta delta = frame->timestamp() - last_time_stamp_;
 240   if (delta.InMilliseconds() < kMinTimeInMsBetweenFrames) {
 241     // We have seen video frames being delivered from camera devices back to
 242     // back. The simple AR filter for frame rate calculation is too short to
 243     // handle that. http://crbug/394315
 244     // TODO(perkj): Can we come up with a way to fix the times stamps and the
 245     // timing when frames are delivered so all frames can be used?
 246     // The time stamps are generated by Chrome and not the actual device.
 247     // Most likely the back to back problem is caused by software and not the
 248     // actual camera.
 249     DVLOG(3) << "Drop frame since delta time since previous frame is "
 250              << delta.InMilliseconds() << "ms.";
 251     return true;
 252   }
 253   last_time_stamp_ = frame->timestamp();
 254   if (delta == last_time_stamp_)  // First received frame.
 255     return false;
 256   // Calculate the frame rate using a simple AR filter.
 257   // Use a simple filter with 0.1 weight of the current sample.
 258   frame_rate_ = 100 / delta.InMillisecondsF() + 0.9 * frame_rate_;
 259
 260   // Prefer to not drop frames.
 261   if (max_frame_rate_ + 0.5f > frame_rate_)
 262     return false;  // Keep this frame.
 263
 264   // The input frame rate is higher than requested.
 265   // Decide if we should keep this frame or drop it.
 266   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 267   if (keep_frame_counter_ >= 1) {
 268     keep_frame_counter_ -= 1;
 269     // Keep the frame.
 270     return false;
 271   }
 272   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 273   return true;
 274 }
 275
 276 void VideoTrackAdapter::
 277 VideoFrameResolutionAdapter::DoDeliverFrame(
 278     const scoped_refptr<media::VideoFrame>& frame,
 279     const media::VideoCaptureFormat& format,
 280     const base::TimeTicks& estimated_capture_time) {
 281   DCHECK(io_thread_checker_.CalledOnValidThread());
 282   for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
 283        it != callbacks_.end(); ++it) {
 284     it->second.Run(frame, format, estimated_capture_time);
 285   }
 286 }
 287
 288 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 289     const MediaStreamVideoTrack* track,
 290     const VideoCaptureDeliverFrameCB& callback) {
 291   DCHECK(io_thread_checker_.CalledOnValidThread());
 292   callbacks_.push_back(std::make_pair(track, callback));
 293 }
 294
 295 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 296     const MediaStreamVideoTrack* track) {
 297   DCHECK(io_thread_checker_.CalledOnValidThread());
 298   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 299   for (; it != callbacks_.end(); ++it) {
 300     if (it->first == track) {
 301       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 302       // render thread since it was added on the main render thread in
 303       // VideoTrackAdapter::AddTrack.
 304       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 305           new VideoCaptureDeliverFrameCB(it->second));
 306       callbacks_.erase(it);
 307       renderer_task_runner_->PostTask(
 308           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 309                                 base::Passed(&callback)));
 310
 311       return;
 312     }
 313   }
 314 }
 315
 316 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 317     const gfx::Size& max_size,
 318     double min_aspect_ratio,
 319     double max_aspect_ratio,
 320     double max_frame_rate) const {
 321   DCHECK(io_thread_checker_.CalledOnValidThread());
 322   return max_frame_size_ == max_size &&
 323       min_aspect_ratio_ == min_aspect_ratio &&
 324       max_aspect_ratio_ == max_aspect_ratio &&
 325       max_frame_rate_ == max_frame_rate;
 326 }
 327
 328 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 329   DCHECK(io_thread_checker_.CalledOnValidThread());
 330   return callbacks_.empty();
 331 }
 332
 333 VideoTrackAdapter::VideoTrackAdapter(
 334     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
 335     : io_message_loop_(io_message_loop),
 336       renderer_task_runner_(base::MessageLoopProxy::current()),
 337       frame_counter_(0),
 338       source_frame_rate_(0.0f) {
 339   DCHECK(io_message_loop_);
 340 }
 341
 342 VideoTrackAdapter::~VideoTrackAdapter() {
 343   DCHECK(adapters_.empty());
 344   UMA_HISTOGRAM_BOOLEAN("Media.VideoTrackAdapter.FramesReceived",
 345                         frame_counter_ > 0);
 346 }
 347
 348 void VideoTrackAdapter::AddTrack(
 349     const MediaStreamVideoTrack* track,
 350     VideoCaptureDeliverFrameCB frame_callback,
 351     int max_width,
 352     int max_height,
 353     double min_aspect_ratio,
 354     double max_aspect_ratio,
 355     double max_frame_rate,
 356     double source_frame_rate,
 357     const OnMutedCallback& on_muted_state_callback) {
 358   DCHECK(thread_checker_.CalledOnValidThread());
 359   // Track monitoring should be scheduled before AddTrackOnIO() so it can find
 360   // |adapters_| empty.
 361   io_message_loop_->PostTask(
 362       FROM_HERE,
 363       base::Bind(&VideoTrackAdapter::StartTrackMonitoringOnIO,
 364                  this, on_muted_state_callback, source_frame_rate));
 365   io_message_loop_->PostTask(
 366       FROM_HERE,
 367       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
 368                  this, track, frame_callback, gfx::Size(max_width, max_height),
 369                  min_aspect_ratio, max_aspect_ratio, max_frame_rate));
 370 }
 371
 372 void VideoTrackAdapter::AddTrackOnIO(
 373     const MediaStreamVideoTrack* track,
 374     VideoCaptureDeliverFrameCB frame_callback,
 375     const gfx::Size& max_frame_size,
 376     double min_aspect_ratio,
 377     double max_aspect_ratio,
 378     double max_frame_rate) {
 379   DCHECK(io_message_loop_->BelongsToCurrentThread());
 380   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 381   for (FrameAdapters::const_iterator it = adapters_.begin();
 382        it != adapters_.end(); ++it) {
 383     if ((*it)->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 384                                 max_aspect_ratio, max_frame_rate)) {
 385       adapter = it->get();
 386       break;
 387     }
 388   }
 389   if (!adapter) {
 390     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 391                                               max_frame_size,
 392                                               min_aspect_ratio,
 393                                               max_aspect_ratio,
 394                                               max_frame_rate);
 395     adapters_.push_back(adapter);
 396   }
 397
 398   adapter->AddCallback(track, frame_callback);
 399 }
 400
 401 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 402   DCHECK(thread_checker_.CalledOnValidThread());
 403   io_message_loop_->PostTask(
 404       FROM_HERE,
 405       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 406 }
 407
 408 void VideoTrackAdapter::StartTrackMonitoringOnIO(
 409     const OnMutedCallback& on_muted_state_callback,
 410     double source_frame_rate) {
 411   DCHECK(io_message_loop_->BelongsToCurrentThread());
 412   // Only trigger monitoring for the first Track.
 413   if (!adapters_.empty())
 414     return;
 415   // If the source does not know the frame rate, set one by default.
 416   if (source_frame_rate == 0.0f)
 417     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 418   source_frame_rate_ = source_frame_rate;
 419   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 420       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 421   io_message_loop_->PostDelayedTask(FROM_HERE,
 422        base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 423                   on_muted_state_callback, frame_counter_),
 424        base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 425                                      source_frame_rate_));
 426 }
 427
 428 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 429   DCHECK(io_message_loop_->BelongsToCurrentThread());
 430   for (FrameAdapters::iterator it = adapters_.begin();
 431        it != adapters_.end(); ++it) {
 432     (*it)->RemoveCallback(track);
 433     if ((*it)->IsEmpty()) {
 434       adapters_.erase(it);
 435       break;
 436     }
 437   }
 438 }
 439
 440 void VideoTrackAdapter::DeliverFrameOnIO(
 441     const scoped_refptr<media::VideoFrame>& frame,
 442     const media::VideoCaptureFormat& format,
 443     const base::TimeTicks& estimated_capture_time) {
 444   DCHECK(io_message_loop_->BelongsToCurrentThread());
 445   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 446   ++frame_counter_;
 447   for (FrameAdapters::iterator it = adapters_.begin();
 448        it != adapters_.end(); ++it) {
 449     (*it)->DeliverFrame(frame, format, estimated_capture_time);
 450   }
 451 }
 452
 453 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 454     const OnMutedCallback& set_muted_state_callback,
 455     uint64 old_frame_counter_snapshot) {
 456   DCHECK(io_message_loop_->BelongsToCurrentThread());
 457   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 458       << "No frames have passed, setting source as Muted.";
 459   set_muted_state_callback.Run(old_frame_counter_snapshot == frame_counter_);
 460
 461   // Rearm the monitoring while there are active Tracks, i.e. as long as the
 462   // owner MediaStreamSource is active.
 463   io_message_loop_->PostDelayedTask(FROM_HERE,
 464       base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 465           set_muted_state_callback, frame_counter_),
 466       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 467                                     source_frame_rate_));
 468 }
 469
 470 }  // namespace content