src/content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/debug/trace_event.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "media/base/bind_to_current_loop.h"
  16 #include "media/base/video_util.h"
  17
  18 namespace content {
  19
  20 namespace {
  21
  22 // Amount of frame intervals to wait before considering the source as muted, for
  23 // the first frame and under normal conditions, respectively. First frame might
  24 // take longer to arrive due to source startup.
  25 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  26 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  27
  28 // Min delta time between two frames allowed without being dropped if a max
  29 // frame rate is specified.
  30 const int kMinTimeInMsBetweenFrames = 5;
  31
  32 // Empty method used for keeping a reference to the original media::VideoFrame
  33 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  34 // The reference to |frame| is kept in the closure that calls this method.
  35 void ReleaseOriginalFrame(
  36     const scoped_refptr<media::VideoFrame>& frame) {
  37 }
  38
  39 void ResetCallbackOnMainRenderThread(
  40     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  41   // |callback| will be deleted when this exits.
  42 }
  43
  44 }  // anonymous namespace
  45
  46 // VideoFrameResolutionAdapter is created on and lives on
  47 // on the IO-thread. It does the resolution adaptation and delivers frames to
  48 // all registered tracks on the IO-thread.
  49 // All method calls must be on the IO-thread.
  50 class VideoTrackAdapter::VideoFrameResolutionAdapter
  51     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  52  public:
  53   VideoFrameResolutionAdapter(
  54       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  55       const gfx::Size& max_size,
  56       double min_aspect_ratio,
  57       double max_aspect_ratio,
  58       double max_frame_rate);
  59
  60   // Add |callback| to receive video frames on the IO-thread.
  61   // |callback| will however be released on the main render thread.
  62   void AddCallback(const MediaStreamVideoTrack* track,
  63                    const VideoCaptureDeliverFrameCB& callback);
  64
  65   // Removes |callback| associated with |track| from receiving video frames if
  66   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  67   // has not been added. The |callback| is released on the main render thread.
  68   void RemoveCallback(const MediaStreamVideoTrack* track);
  69
  70   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  71                     const media::VideoCaptureFormat& format,
  72                     const base::TimeTicks& estimated_capture_time);
  73
  74   // Returns true if all arguments match with the output of this adapter.
  75   bool ConstraintsMatch(const gfx::Size& max_size,
  76                         double min_aspect_ratio,
  77                         double max_aspect_ratio,
  78                         double max_frame_rate) const;
  79
  80   bool IsEmpty() const;
  81
  82  private:
  83   virtual ~VideoFrameResolutionAdapter();
  84   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  85
  86   virtual void DoDeliverFrame(
  87       const scoped_refptr<media::VideoFrame>& frame,
  88       const media::VideoCaptureFormat& format,
  89       const base::TimeTicks& estimated_capture_time);
  90
  91   // Returns |true| if the input frame rate is higher that the requested max
  92   // frame rate and |frame| should be dropped.
  93   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame,
  94                       float source_frame_rate);
  95
  96   // Bound to the IO-thread.
  97   base::ThreadChecker io_thread_checker_;
  98
  99   // The task runner where we will release VideoCaptureDeliverFrameCB
 100   // registered in AddCallback.
 101   scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 102
 103   gfx::Size max_frame_size_;
 104   double min_aspect_ratio_;
 105   double max_aspect_ratio_;
 106
 107   double frame_rate_;
 108   base::TimeDelta last_time_stamp_;
 109   double max_frame_rate_;
 110   double keep_frame_counter_;
 111
 112   typedef std::pair<const void*, VideoCaptureDeliverFrameCB>
 113       VideoIdCallbackPair;
 114   std::vector<VideoIdCallbackPair> callbacks_;
 115
 116   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 117 };
 118
 119 VideoTrackAdapter::
 120 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 121     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 122     const gfx::Size& max_size,
 123     double min_aspect_ratio,
 124     double max_aspect_ratio,
 125     double max_frame_rate)
 126     : renderer_task_runner_(render_message_loop),
 127       max_frame_size_(max_size),
 128       min_aspect_ratio_(min_aspect_ratio),
 129       max_aspect_ratio_(max_aspect_ratio),
 130       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 131       max_frame_rate_(max_frame_rate),
 132       keep_frame_counter_(0.0f) {
 133   DCHECK(renderer_task_runner_.get());
 134   DCHECK(io_thread_checker_.CalledOnValidThread());
 135   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 136   CHECK_NE(0, max_aspect_ratio_);
 137   DVLOG(3) << "VideoFrameResolutionAdapter("
 138           << "{ max_width =" << max_frame_size_.width() << "}, "
 139           << "{ max_height =" << max_frame_size_.height() << "}, "
 140           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 141           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 142           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 143 }
 144
 145 VideoTrackAdapter::
 146 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 147   DCHECK(io_thread_checker_.CalledOnValidThread());
 148   DCHECK(callbacks_.empty());
 149 }
 150
 151 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 152     const scoped_refptr<media::VideoFrame>& frame,
 153     const media::VideoCaptureFormat& format,
 154     const base::TimeTicks& estimated_capture_time) {
 155   DCHECK(io_thread_checker_.CalledOnValidThread());
 156
 157   if (MaybeDropFrame(frame, format.frame_rate))
 158     return;
 159
 160   // TODO(perkj): Allow cropping / scaling of textures once
 161   // http://crbug/362521 is fixed.
 162   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
 163     DoDeliverFrame(frame, format, estimated_capture_time);
 164     return;
 165   }
 166   scoped_refptr<media::VideoFrame> video_frame(frame);
 167   double input_ratio =
 168       static_cast<double>(frame->natural_size().width()) /
 169       frame->natural_size().height();
 170
 171   // If |frame| has larger width or height than requested, or the aspect ratio
 172   // does not match the requested, we want to create a wrapped version of this
 173   // frame with a size that fulfills the constraints.
 174   if (frame->natural_size().width() > max_frame_size_.width() ||
 175       frame->natural_size().height() > max_frame_size_.height() ||
 176       input_ratio > max_aspect_ratio_ ||
 177       input_ratio < min_aspect_ratio_) {
 178     int desired_width = std::min(max_frame_size_.width(),
 179                                  frame->natural_size().width());
 180     int desired_height = std::min(max_frame_size_.height(),
 181                                   frame->natural_size().height());
 182
 183     double resulting_ratio =
 184         static_cast<double>(desired_width) / desired_height;
 185     double requested_ratio = resulting_ratio;
 186
 187     if (requested_ratio > max_aspect_ratio_)
 188       requested_ratio = max_aspect_ratio_;
 189     else if (requested_ratio < min_aspect_ratio_)
 190       requested_ratio = min_aspect_ratio_;
 191
 192     if (resulting_ratio < requested_ratio) {
 193       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 194                                         requested_ratio);
 195       // Make sure we scale to an even height to avoid rounding errors
 196       desired_height = (desired_height + 1) & ~1;
 197     } else if (resulting_ratio > requested_ratio) {
 198       desired_width = static_cast<int>((desired_width * requested_ratio) /
 199                                        resulting_ratio);
 200       // Make sure we scale to an even width to avoid rounding errors.
 201       desired_width = (desired_width + 1) & ~1;
 202     }
 203
 204     gfx::Size desired_size(desired_width, desired_height);
 205
 206     // Get the largest centered rectangle with the same aspect ratio of
 207     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 208     // This will be the rect we need to crop the original frame to.
 209     // From this rect, the original frame can be scaled down to |desired_size|.
 210     gfx::Rect region_in_frame =
 211         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 212
 213     video_frame = media::VideoFrame::WrapVideoFrame(
 214         frame,
 215         region_in_frame,
 216         desired_size,
 217         base::Bind(&ReleaseOriginalFrame, frame));
 218
 219     DVLOG(3) << "desired size  " << desired_size.ToString()
 220              << " output natural size "
 221              << video_frame->natural_size().ToString()
 222              << " output visible rect  "
 223              << video_frame->visible_rect().ToString();
 224   }
 225   DoDeliverFrame(video_frame, format, estimated_capture_time);
 226 }
 227
 228 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 229     const scoped_refptr<media::VideoFrame>& frame,
 230     float source_frame_rate) {
 231   DCHECK(io_thread_checker_.CalledOnValidThread());
 232
 233   // Do not drop frames if max frame rate hasn't been specified or the source
 234   // frame rate is known and is lower than max.
 235   if (max_frame_rate_ == 0.0f ||
 236       (source_frame_rate > 0 &&
 237           source_frame_rate <= max_frame_rate_)) {
 238     return false;
 239   }
 240
 241   base::TimeDelta delta = frame->timestamp() - last_time_stamp_;
 242   if (delta.InMilliseconds() < kMinTimeInMsBetweenFrames) {
 243     // We have seen video frames being delivered from camera devices back to
 244     // back. The simple AR filter for frame rate calculation is too short to
 245     // handle that. http://crbug/394315
 246     // TODO(perkj): Can we come up with a way to fix the times stamps and the
 247     // timing when frames are delivered so all frames can be used?
 248     // The time stamps are generated by Chrome and not the actual device.
 249     // Most likely the back to back problem is caused by software and not the
 250     // actual camera.
 251     DVLOG(3) << "Drop frame since delta time since previous frame is "
 252              << delta.InMilliseconds() << "ms.";
 253     return true;
 254   }
 255   last_time_stamp_ = frame->timestamp();
 256   if (delta == last_time_stamp_)  // First received frame.
 257     return false;
 258   // Calculate the frame rate using a simple AR filter.
 259   // Use a simple filter with 0.1 weight of the current sample.
 260   frame_rate_ = 100 / delta.InMillisecondsF() + 0.9 * frame_rate_;
 261
 262   // Prefer to not drop frames.
 263   if (max_frame_rate_ + 0.5f > frame_rate_)
 264     return false;  // Keep this frame.
 265
 266   // The input frame rate is higher than requested.
 267   // Decide if we should keep this frame or drop it.
 268   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 269   if (keep_frame_counter_ >= 1) {
 270     keep_frame_counter_ -= 1;
 271     // Keep the frame.
 272     return false;
 273   }
 274   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 275   return true;
 276 }
 277
 278 void VideoTrackAdapter::
 279 VideoFrameResolutionAdapter::DoDeliverFrame(
 280     const scoped_refptr<media::VideoFrame>& frame,
 281     const media::VideoCaptureFormat& format,
 282     const base::TimeTicks& estimated_capture_time) {
 283   DCHECK(io_thread_checker_.CalledOnValidThread());
 284   for (std::vector<VideoIdCallbackPair>::const_iterator it = callbacks_.begin();
 285        it != callbacks_.end(); ++it) {
 286     it->second.Run(frame, format, estimated_capture_time);
 287   }
 288 }
 289
 290 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 291     const MediaStreamVideoTrack* track,
 292     const VideoCaptureDeliverFrameCB& callback) {
 293   DCHECK(io_thread_checker_.CalledOnValidThread());
 294   callbacks_.push_back(std::make_pair(track, callback));
 295 }
 296
 297 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 298     const MediaStreamVideoTrack* track) {
 299   DCHECK(io_thread_checker_.CalledOnValidThread());
 300   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 301   for (; it != callbacks_.end(); ++it) {
 302     if (it->first == track) {
 303       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 304       // render thread since it was added on the main render thread in
 305       // VideoTrackAdapter::AddTrack.
 306       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 307           new VideoCaptureDeliverFrameCB(it->second));
 308       callbacks_.erase(it);
 309       renderer_task_runner_->PostTask(
 310           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 311                                 base::Passed(&callback)));
 312
 313       return;
 314     }
 315   }
 316 }
 317
 318 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 319     const gfx::Size& max_size,
 320     double min_aspect_ratio,
 321     double max_aspect_ratio,
 322     double max_frame_rate) const {
 323   DCHECK(io_thread_checker_.CalledOnValidThread());
 324   return max_frame_size_ == max_size &&
 325       min_aspect_ratio_ == min_aspect_ratio &&
 326       max_aspect_ratio_ == max_aspect_ratio &&
 327       max_frame_rate_ == max_frame_rate;
 328 }
 329
 330 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 331   DCHECK(io_thread_checker_.CalledOnValidThread());
 332   return callbacks_.empty();
 333 }
 334
 335 VideoTrackAdapter::VideoTrackAdapter(
 336     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
 337     : io_message_loop_(io_message_loop),
 338       renderer_task_runner_(base::MessageLoopProxy::current()),
 339       monitoring_frame_rate_(false),
 340       muted_state_(false),
 341       frame_counter_(0),
 342       source_frame_rate_(0.0f) {
 343   DCHECK(io_message_loop_.get());
 344 }
 345
 346 VideoTrackAdapter::~VideoTrackAdapter() {
 347   DCHECK(adapters_.empty());
 348 }
 349
 350 void VideoTrackAdapter::AddTrack(
 351     const MediaStreamVideoTrack* track,
 352     VideoCaptureDeliverFrameCB frame_callback,
 353     int max_width,
 354     int max_height,
 355     double min_aspect_ratio,
 356     double max_aspect_ratio,
 357     double max_frame_rate) {
 358   DCHECK(thread_checker_.CalledOnValidThread());
 359
 360   io_message_loop_->PostTask(
 361       FROM_HERE,
 362       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
 363                  this, track, frame_callback, gfx::Size(max_width, max_height),
 364                  min_aspect_ratio, max_aspect_ratio, max_frame_rate));
 365 }
 366
 367 void VideoTrackAdapter::AddTrackOnIO(
 368     const MediaStreamVideoTrack* track,
 369     VideoCaptureDeliverFrameCB frame_callback,
 370     const gfx::Size& max_frame_size,
 371     double min_aspect_ratio,
 372     double max_aspect_ratio,
 373     double max_frame_rate) {
 374   DCHECK(io_message_loop_->BelongsToCurrentThread());
 375   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 376   for (FrameAdapters::const_iterator it = adapters_.begin();
 377        it != adapters_.end(); ++it) {
 378     if ((*it)->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 379                                 max_aspect_ratio, max_frame_rate)) {
 380       adapter = it->get();
 381       break;
 382     }
 383   }
 384   if (!adapter.get()) {
 385     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 386                                               max_frame_size,
 387                                               min_aspect_ratio,
 388                                               max_aspect_ratio,
 389                                               max_frame_rate);
 390     adapters_.push_back(adapter);
 391   }
 392
 393   adapter->AddCallback(track, frame_callback);
 394 }
 395
 396 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 397   DCHECK(thread_checker_.CalledOnValidThread());
 398   io_message_loop_->PostTask(
 399       FROM_HERE,
 400       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 401 }
 402
 403 void VideoTrackAdapter::StartFrameMonitoring(
 404     double source_frame_rate,
 405     const OnMutedCallback& on_muted_callback) {
 406   DCHECK(thread_checker_.CalledOnValidThread());
 407
 408   VideoTrackAdapter::OnMutedCallback bound_on_muted_callback =
 409       media::BindToCurrentLoop(on_muted_callback);
 410
 411   io_message_loop_->PostTask(
 412       FROM_HERE,
 413       base::Bind(&VideoTrackAdapter::StartFrameMonitoringOnIO,
 414                  this, bound_on_muted_callback, source_frame_rate));
 415 }
 416
 417 void VideoTrackAdapter::StartFrameMonitoringOnIO(
 418     const OnMutedCallback& on_muted_callback,
 419     double source_frame_rate) {
 420   DCHECK(io_message_loop_->BelongsToCurrentThread());
 421   DCHECK(!monitoring_frame_rate_);
 422
 423   monitoring_frame_rate_ = true;
 424
 425   // If the source does not know the frame rate, set one by default.
 426   if (source_frame_rate == 0.0f)
 427     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 428   source_frame_rate_ = source_frame_rate;
 429   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 430       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 431   io_message_loop_->PostDelayedTask(FROM_HERE,
 432        base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 433                   on_muted_callback, frame_counter_),
 434        base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 435                                      source_frame_rate_));
 436 }
 437
 438 void VideoTrackAdapter::StopFrameMonitoring() {
 439   DCHECK(thread_checker_.CalledOnValidThread());
 440   io_message_loop_->PostTask(
 441       FROM_HERE,
 442       base::Bind(&VideoTrackAdapter::StopFrameMonitoringOnIO, this));
 443 }
 444
 445 void VideoTrackAdapter::StopFrameMonitoringOnIO() {
 446   DCHECK(io_message_loop_->BelongsToCurrentThread());
 447   monitoring_frame_rate_ = false;
 448 }
 449
 450 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 451   DCHECK(io_message_loop_->BelongsToCurrentThread());
 452   for (FrameAdapters::iterator it = adapters_.begin();
 453        it != adapters_.end(); ++it) {
 454     (*it)->RemoveCallback(track);
 455     if ((*it)->IsEmpty()) {
 456       adapters_.erase(it);
 457       break;
 458     }
 459   }
 460 }
 461
 462 void VideoTrackAdapter::DeliverFrameOnIO(
 463     const scoped_refptr<media::VideoFrame>& frame,
 464     const media::VideoCaptureFormat& format,
 465     const base::TimeTicks& estimated_capture_time) {
 466   DCHECK(io_message_loop_->BelongsToCurrentThread());
 467   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 468   ++frame_counter_;
 469   for (FrameAdapters::iterator it = adapters_.begin();
 470        it != adapters_.end(); ++it) {
 471     (*it)->DeliverFrame(frame, format, estimated_capture_time);
 472   }
 473 }
 474
 475 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 476     const OnMutedCallback& set_muted_state_callback,
 477     uint64 old_frame_counter_snapshot) {
 478   DCHECK(io_message_loop_->BelongsToCurrentThread());
 479
 480   if (!monitoring_frame_rate_)
 481     return;
 482
 483   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 484       << "No frames have passed, setting source as Muted.";
 485
 486   bool muted_state = old_frame_counter_snapshot == frame_counter_;
 487   if (muted_state_ != muted_state) {
 488     set_muted_state_callback.Run(muted_state);
 489     muted_state_ = muted_state;
 490   }
 491
 492   io_message_loop_->PostDelayedTask(FROM_HERE,
 493       base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 494           set_muted_state_callback, frame_counter_),
 495       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 496                                     source_frame_rate_));
 497 }
 498
 499 }  // namespace content