src/content/browser/media/capture/desktop_capture_device.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device.h"
   6
   7 #include "base/bind.h"
   8 #include "base/location.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/synchronization/lock.h"
  13 #include "base/threading/thread.h"
  14 #include "base/timer/timer.h"
  15 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "content/public/browser/desktop_media_id.h"
  18 #include "content/public/browser/power_save_blocker.h"
  19 #include "media/base/video_util.h"
  20 #include "third_party/libyuv/include/libyuv/scale_argb.h"
  21 #include "third_party/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h"
  22 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
  23 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  25 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
  26 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
  27 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
  28
  29 namespace content {
  30
  31 namespace {
  32
  33 // Maximum CPU time percentage of a single core that can be consumed for desktop
  34 // capturing. This means that on systems where screen scraping is slow we may
  35 // need to capture at frame rate lower than requested. This is necessary to keep
  36 // UI responsive.
  37 const int kMaximumCpuConsumptionPercentage = 50;
  38
  39 webrtc::DesktopRect ComputeLetterboxRect(
  40     const webrtc::DesktopSize& max_size,
  41     const webrtc::DesktopSize& source_size) {
  42   gfx::Rect result = media::ComputeLetterboxRegion(
  43       gfx::Rect(0, 0, max_size.width(), max_size.height()),
  44       gfx::Size(source_size.width(), source_size.height()));
  45   return webrtc::DesktopRect::MakeLTRB(
  46       result.x(), result.y(), result.right(), result.bottom());
  47 }
  48
  49 }  // namespace
  50
  51 class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback {
  52  public:
  53   Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  54        scoped_ptr<webrtc::DesktopCapturer> capturer,
  55        DesktopMediaID::Type type);
  56   ~Core() override;
  57
  58   // Implementation of VideoCaptureDevice methods.
  59   void AllocateAndStart(const media::VideoCaptureParams& params,
  60                         scoped_ptr<Client> client);
  61
  62   void SetNotificationWindowId(gfx::NativeViewId window_id);
  63
  64  private:
  65
  66   // webrtc::DesktopCapturer::Callback interface
  67   webrtc::SharedMemory* CreateSharedMemory(size_t size) override;
  68   void OnCaptureCompleted(webrtc::DesktopFrame* frame) override;
  69
  70   // Chooses new output properties based on the supplied source size and the
  71   // properties requested to Allocate(), and dispatches OnFrameInfo[Changed]
  72   // notifications.
  73   void RefreshCaptureFormat(const webrtc::DesktopSize& frame_size);
  74
  75   // Method that is scheduled on |task_runner_| to be called on regular interval
  76   // to capture a frame.
  77   void OnCaptureTimer();
  78
  79   // Captures a frame and schedules timer for the next one.
  80   void CaptureFrameAndScheduleNext();
  81
  82   // Captures a single frame.
  83   void DoCapture();
  84
  85   // Task runner used for capturing operations.
  86   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  87
  88   // The underlying DesktopCapturer instance used to capture frames.
  89   scoped_ptr<webrtc::DesktopCapturer> desktop_capturer_;
  90
  91   // The device client which proxies device events to the controller. Accessed
  92   // on the task_runner_ thread.
  93   scoped_ptr<Client> client_;
  94
  95   // Requested video capture format (width, height, frame rate, etc).
  96   media::VideoCaptureParams requested_params_;
  97
  98   // Actual video capture format being generated.
  99   media::VideoCaptureFormat capture_format_;
 100
 101   // Size of frame most recently captured from the source.
 102   webrtc::DesktopSize previous_frame_size_;
 103
 104   // DesktopFrame into which captured frames are down-scaled and/or letterboxed,
 105   // depending upon the caller's requested capture capabilities. If frames can
 106   // be returned to the caller directly then this is NULL.
 107   scoped_ptr<webrtc::DesktopFrame> output_frame_;
 108
 109   // Sub-rectangle of |output_frame_| into which the source will be scaled
 110   // and/or letterboxed.
 111   webrtc::DesktopRect output_rect_;
 112
 113   // Timer used to capture the frame.
 114   base::OneShotTimer<Core> capture_timer_;
 115
 116   // True when waiting for |desktop_capturer_| to capture current frame.
 117   bool capture_in_progress_;
 118
 119   // True if the first capture call has returned. Used to log the first capture
 120   // result.
 121   bool first_capture_returned_;
 122
 123   // The type of the capturer.
 124   DesktopMediaID::Type capturer_type_;
 125
 126   scoped_ptr<webrtc::BasicDesktopFrame> black_frame_;
 127
 128   // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
 129   // screen from sleeping for the drive-by web.
 130   scoped_ptr<PowerSaveBlocker> power_save_blocker_;
 131
 132   DISALLOW_COPY_AND_ASSIGN(Core);
 133 };
 134
 135 DesktopCaptureDevice::Core::Core(
 136     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
 137     scoped_ptr<webrtc::DesktopCapturer> capturer,
 138     DesktopMediaID::Type type)
 139     : task_runner_(task_runner),
 140       desktop_capturer_(capturer.Pass()),
 141       capture_in_progress_(false),
 142       first_capture_returned_(false),
 143       capturer_type_(type) {
 144 }
 145
 146 DesktopCaptureDevice::Core::~Core() {
 147   DCHECK(task_runner_->BelongsToCurrentThread());
 148   client_.reset();
 149   output_frame_.reset();
 150   previous_frame_size_.set(0, 0);
 151   desktop_capturer_.reset();
 152 }
 153
 154 void DesktopCaptureDevice::Core::AllocateAndStart(
 155     const media::VideoCaptureParams& params,
 156     scoped_ptr<Client> client) {
 157   DCHECK(task_runner_->BelongsToCurrentThread());
 158   DCHECK_GT(params.requested_format.frame_size.GetArea(), 0);
 159   DCHECK_GT(params.requested_format.frame_rate, 0);
 160   DCHECK(desktop_capturer_);
 161   DCHECK(client.get());
 162   DCHECK(!client_.get());
 163
 164   client_ = client.Pass();
 165   requested_params_ = params;
 166
 167   capture_format_ = requested_params_.requested_format;
 168
 169   // This capturer always outputs ARGB, non-interlaced.
 170   capture_format_.pixel_format = media::PIXEL_FORMAT_ARGB;
 171
 172   power_save_blocker_.reset(PowerSaveBlocker::Create(
 173       PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 174       "DesktopCaptureDevice is running").release());
 175
 176   desktop_capturer_->Start(this);
 177
 178   CaptureFrameAndScheduleNext();
 179 }
 180
 181 void DesktopCaptureDevice::Core::SetNotificationWindowId(
 182     gfx::NativeViewId window_id) {
 183   DCHECK(task_runner_->BelongsToCurrentThread());
 184   DCHECK(window_id);
 185   desktop_capturer_->SetExcludedWindow(window_id);
 186 }
 187
 188 webrtc::SharedMemory*
 189 DesktopCaptureDevice::Core::CreateSharedMemory(size_t size) {
 190   return NULL;
 191 }
 192
 193 void DesktopCaptureDevice::Core::OnCaptureCompleted(
 194     webrtc::DesktopFrame* frame) {
 195   DCHECK(task_runner_->BelongsToCurrentThread());
 196   DCHECK(capture_in_progress_);
 197
 198   if (!first_capture_returned_) {
 199     first_capture_returned_ = true;
 200     if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 201       IncrementDesktopCaptureCounter(frame ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 202                                            : FIRST_SCREEN_CAPTURE_FAILED);
 203     } else {
 204       IncrementDesktopCaptureCounter(frame ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 205                                            : FIRST_WINDOW_CAPTURE_FAILED);
 206     }
 207   }
 208
 209   capture_in_progress_ = false;
 210
 211   if (!frame) {
 212     std::string log("Failed to capture a frame.");
 213     LOG(ERROR) << log;
 214     client_->OnError(log);
 215     return;
 216   }
 217
 218   if (!client_)
 219     return;
 220
 221   base::TimeDelta capture_time(
 222       base::TimeDelta::FromMilliseconds(frame->capture_time_ms()));
 223
 224   // The two UMA_ blocks must be put in its own scope since it creates a static
 225   // variable which expected constant histogram name.
 226   if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 227     UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
 228   } else {
 229     UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
 230   }
 231
 232   scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
 233
 234   // On OSX We receive a 1x1 frame when the shared window is minimized. It
 235   // cannot be subsampled to I420 and will be dropped downstream. So we replace
 236   // it with a black frame to avoid the video appearing frozen at the last
 237   // frame.
 238   if (frame->size().width() == 1 || frame->size().height() == 1) {
 239     if (!black_frame_.get()) {
 240       black_frame_.reset(
 241           new webrtc::BasicDesktopFrame(
 242               webrtc::DesktopSize(capture_format_.frame_size.width(),
 243                                   capture_format_.frame_size.height())));
 244       memset(black_frame_->data(),
 245              0,
 246              black_frame_->stride() * black_frame_->size().height());
 247     }
 248     owned_frame.reset();
 249     frame = black_frame_.get();
 250   }
 251
 252   // Handle initial frame size and size changes.
 253   RefreshCaptureFormat(frame->size());
 254
 255   webrtc::DesktopSize output_size(capture_format_.frame_size.width(),
 256                                   capture_format_.frame_size.height());
 257   size_t output_bytes = output_size.width() * output_size.height() *
 258       webrtc::DesktopFrame::kBytesPerPixel;
 259   const uint8_t* output_data = NULL;
 260   scoped_ptr<uint8_t[]> flipped_frame_buffer;
 261
 262   if (frame->size().equals(output_size)) {
 263     // If the captured frame matches the output size, we can return the pixel
 264     // data directly, without scaling.
 265     output_data = frame->data();
 266
 267     // If the |frame| generated by the screen capturer is inverted then we need
 268     // to flip |frame|.
 269     // This happens only on a specific platform. Refer to crbug.com/306876.
 270     if (frame->stride() < 0) {
 271       int height = frame->size().height();
 272       int bytes_per_row =
 273           frame->size().width() * webrtc::DesktopFrame::kBytesPerPixel;
 274       flipped_frame_buffer.reset(new uint8_t[output_bytes]);
 275       uint8_t* dest = flipped_frame_buffer.get();
 276       for (int row = 0; row < height; ++row) {
 277         memcpy(dest, output_data, bytes_per_row);
 278         dest += bytes_per_row;
 279         output_data += frame->stride();
 280       }
 281       output_data = flipped_frame_buffer.get();
 282     }
 283   } else {
 284     // Otherwise we need to down-scale and/or letterbox to the target format.
 285
 286     // Allocate a buffer of the correct size to scale the frame into.
 287     // |output_frame_| is cleared whenever |output_rect_| changes, so we don't
 288     // need to worry about clearing out stale pixel data in letterboxed areas.
 289     if (!output_frame_) {
 290       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 291       memset(output_frame_->data(), 0, output_bytes);
 292     }
 293     DCHECK(output_frame_->size().equals(output_size));
 294
 295     // TODO(wez): Optimize this to scale only changed portions of the output,
 296     // using ARGBScaleClip().
 297     uint8_t* output_rect_data = output_frame_->data() +
 298         output_frame_->stride() * output_rect_.top() +
 299         webrtc::DesktopFrame::kBytesPerPixel * output_rect_.left();
 300     libyuv::ARGBScale(frame->data(), frame->stride(),
 301                       frame->size().width(), frame->size().height(),
 302                       output_rect_data, output_frame_->stride(),
 303                       output_rect_.width(), output_rect_.height(),
 304                       libyuv::kFilterBilinear);
 305     output_data = output_frame_->data();
 306   }
 307
 308   client_->OnIncomingCapturedData(
 309       output_data, output_bytes, capture_format_, 0, base::TimeTicks::Now());
 310 }
 311
 312 void DesktopCaptureDevice::Core::RefreshCaptureFormat(
 313     const webrtc::DesktopSize& frame_size) {
 314   if (previous_frame_size_.equals(frame_size))
 315     return;
 316
 317   // Clear the output frame, if any, since it will either need resizing, or
 318   // clearing of stale data in letterbox areas, anyway.
 319   output_frame_.reset();
 320
 321   if (previous_frame_size_.is_empty() ||
 322       requested_params_.resolution_change_policy ==
 323       media::RESOLUTION_POLICY_DYNAMIC_WITHIN_LIMIT) {
 324     // If this is the first frame, or the receiver supports variable resolution
 325     // then determine the output size by treating the requested width & height
 326     // as maxima.
 327     if (frame_size.width() >
 328             requested_params_.requested_format.frame_size.width() ||
 329         frame_size.height() >
 330             requested_params_.requested_format.frame_size.height()) {
 331       output_rect_ = ComputeLetterboxRect(
 332           webrtc::DesktopSize(
 333               requested_params_.requested_format.frame_size.width(),
 334               requested_params_.requested_format.frame_size.height()),
 335           frame_size);
 336       output_rect_.Translate(-output_rect_.left(), -output_rect_.top());
 337     } else {
 338       output_rect_ = webrtc::DesktopRect::MakeSize(frame_size);
 339     }
 340     capture_format_.frame_size.SetSize(output_rect_.width(),
 341                                        output_rect_.height());
 342   } else {
 343     // Otherwise the output frame size cannot change, so just scale and
 344     // letterbox.
 345     output_rect_ = ComputeLetterboxRect(
 346         webrtc::DesktopSize(capture_format_.frame_size.width(),
 347                             capture_format_.frame_size.height()),
 348         frame_size);
 349   }
 350
 351   previous_frame_size_ = frame_size;
 352 }
 353
 354 void DesktopCaptureDevice::Core::OnCaptureTimer() {
 355   DCHECK(task_runner_->BelongsToCurrentThread());
 356
 357   if (!client_)
 358     return;
 359
 360   CaptureFrameAndScheduleNext();
 361 }
 362
 363 void DesktopCaptureDevice::Core::CaptureFrameAndScheduleNext() {
 364   DCHECK(task_runner_->BelongsToCurrentThread());
 365
 366   base::TimeTicks started_time = base::TimeTicks::Now();
 367   DoCapture();
 368   base::TimeDelta last_capture_duration = base::TimeTicks::Now() - started_time;
 369
 370   // Limit frame-rate to reduce CPU consumption.
 371   base::TimeDelta capture_period = std::max(
 372     (last_capture_duration * 100) / kMaximumCpuConsumptionPercentage,
 373     base::TimeDelta::FromSeconds(1) / capture_format_.frame_rate);
 374
 375   // Schedule a task for the next frame.
 376   capture_timer_.Start(FROM_HERE, capture_period - last_capture_duration,
 377                        this, &Core::OnCaptureTimer);
 378 }
 379
 380 void DesktopCaptureDevice::Core::DoCapture() {
 381   DCHECK(task_runner_->BelongsToCurrentThread());
 382   DCHECK(!capture_in_progress_);
 383
 384   capture_in_progress_ = true;
 385   desktop_capturer_->Capture(webrtc::DesktopRegion());
 386
 387   // Currently only synchronous implementations of DesktopCapturer are
 388   // supported.
 389   DCHECK(!capture_in_progress_);
 390 }
 391
 392 // static
 393 scoped_ptr<media::VideoCaptureDevice> DesktopCaptureDevice::Create(
 394     const DesktopMediaID& source) {
 395   webrtc::DesktopCaptureOptions options =
 396       webrtc::DesktopCaptureOptions::CreateDefault();
 397   // Leave desktop effects enabled during WebRTC captures.
 398   options.set_disable_effects(false);
 399
 400   scoped_ptr<webrtc::DesktopCapturer> capturer;
 401
 402   switch (source.type) {
 403     case DesktopMediaID::TYPE_SCREEN: {
 404 #if defined(OS_WIN)
 405       options.set_allow_use_magnification_api(true);
 406 #endif
 407       scoped_ptr<webrtc::ScreenCapturer> screen_capturer(
 408           webrtc::ScreenCapturer::Create(options));
 409       if (screen_capturer && screen_capturer->SelectScreen(source.id)) {
 410         capturer.reset(new webrtc::DesktopAndCursorComposer(
 411             screen_capturer.release(),
 412             webrtc::MouseCursorMonitor::CreateForScreen(options, source.id)));
 413         IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
 414       }
 415       break;
 416     }
 417
 418     case DesktopMediaID::TYPE_WINDOW: {
 419       scoped_ptr<webrtc::WindowCapturer> window_capturer(
 420           webrtc::WindowCapturer::Create(options));
 421       if (window_capturer && window_capturer->SelectWindow(source.id)) {
 422         window_capturer->BringSelectedWindowToFront();
 423         capturer.reset(new webrtc::DesktopAndCursorComposer(
 424             window_capturer.release(),
 425             webrtc::MouseCursorMonitor::CreateForWindow(options, source.id)));
 426         IncrementDesktopCaptureCounter(WINDOW_CAPTURER_CREATED);
 427       }
 428       break;
 429     }
 430
 431     default: {
 432       NOTREACHED();
 433     }
 434   }
 435
 436   scoped_ptr<media::VideoCaptureDevice> result;
 437   if (capturer)
 438     result.reset(new DesktopCaptureDevice(capturer.Pass(), source.type));
 439
 440   return result.Pass();
 441 }
 442
 443 DesktopCaptureDevice::~DesktopCaptureDevice() {
 444   DCHECK(!core_);
 445 }
 446
 447 void DesktopCaptureDevice::AllocateAndStart(
 448     const media::VideoCaptureParams& params,
 449     scoped_ptr<Client> client) {
 450   thread_.message_loop_proxy()->PostTask(
 451       FROM_HERE,
 452       base::Bind(&Core::AllocateAndStart, base::Unretained(core_.get()), params,
 453                  base::Passed(&client)));
 454 }
 455
 456 void DesktopCaptureDevice::StopAndDeAllocate() {
 457   if (core_) {
 458     thread_.message_loop_proxy()->DeleteSoon(FROM_HERE, core_.release());
 459     thread_.Stop();
 460   }
 461 }
 462
 463 void DesktopCaptureDevice::SetNotificationWindowId(
 464     gfx::NativeViewId window_id) {
 465   thread_.message_loop_proxy()->PostTask(
 466       FROM_HERE,
 467       base::Bind(&Core::SetNotificationWindowId, base::Unretained(core_.get()),
 468                  window_id));
 469 }
 470
 471 DesktopCaptureDevice::DesktopCaptureDevice(
 472     scoped_ptr<webrtc::DesktopCapturer> capturer,
 473     DesktopMediaID::Type type)
 474     : thread_("desktopCaptureThread") {
 475 #if defined(OS_WIN)
 476   // On Windows the thread must be a UI thread.
 477   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_UI;
 478 #else
 479   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_DEFAULT;
 480 #endif
 481
 482   thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
 483
 484   core_.reset(new Core(thread_.message_loop_proxy(), capturer.Pass(), type));
 485 }
 486
 487 }  // namespace content