1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/media/capture/desktop_capture_device.h"
8 #include "base/location.h"
9 #include "base/logging.h"
10 #include "base/metrics/histogram.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "base/synchronization/lock.h"
13 #include "base/threading/thread.h"
14 #include "base/timer/timer.h"
15 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
16 #include "content/public/browser/browser_thread.h"
17 #include "content/public/browser/desktop_media_id.h"
18 #include "media/base/video_util.h"
19 #include "third_party/libyuv/include/libyuv/scale_argb.h"
20 #include "third_party/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h"
21 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
22 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
23 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
24 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
25 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
26 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
32 // Maximum CPU time percentage of a single core that can be consumed for desktop
33 // capturing. This means that on systems where screen scraping is slow we may
34 // need to capture at frame rate lower than requested. This is necessary to keep
36 const int kMaximumCpuConsumptionPercentage = 50;
38 webrtc::DesktopRect ComputeLetterboxRect(
39 const webrtc::DesktopSize& max_size,
40 const webrtc::DesktopSize& source_size) {
41 gfx::Rect result = media::ComputeLetterboxRegion(
42 gfx::Rect(0, 0, max_size.width(), max_size.height()),
43 gfx::Size(source_size.width(), source_size.height()));
44 return webrtc::DesktopRect::MakeLTRB(
45 result.x(), result.y(), result.right(), result.bottom());
50 class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback {
52 Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
53 scoped_ptr<webrtc::DesktopCapturer> capturer,
54 DesktopMediaID::Type type);
57 // Implementation of VideoCaptureDevice methods.
58 void AllocateAndStart(const media::VideoCaptureParams& params,
59 scoped_ptr<Client> client);
61 void SetNotificationWindowId(gfx::NativeViewId window_id);
65 // webrtc::DesktopCapturer::Callback interface
66 virtual webrtc::SharedMemory* CreateSharedMemory(size_t size) OVERRIDE;
67 virtual void OnCaptureCompleted(webrtc::DesktopFrame* frame) OVERRIDE;
69 // Chooses new output properties based on the supplied source size and the
70 // properties requested to Allocate(), and dispatches OnFrameInfo[Changed]
72 void RefreshCaptureFormat(const webrtc::DesktopSize& frame_size);
74 // Method that is scheduled on |task_runner_| to be called on regular interval
75 // to capture a frame.
76 void OnCaptureTimer();
78 // Captures a frame and schedules timer for the next one.
79 void CaptureFrameAndScheduleNext();
81 // Captures a single frame.
84 // Task runner used for capturing operations.
85 scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
87 // The underlying DesktopCapturer instance used to capture frames.
88 scoped_ptr<webrtc::DesktopCapturer> desktop_capturer_;
90 // The device client which proxies device events to the controller. Accessed
91 // on the task_runner_ thread.
92 scoped_ptr<Client> client_;
94 // Requested video capture format (width, height, frame rate, etc).
95 media::VideoCaptureParams requested_params_;
97 // Actual video capture format being generated.
98 media::VideoCaptureFormat capture_format_;
100 // Size of frame most recently captured from the source.
101 webrtc::DesktopSize previous_frame_size_;
103 // DesktopFrame into which captured frames are down-scaled and/or letterboxed,
104 // depending upon the caller's requested capture capabilities. If frames can
105 // be returned to the caller directly then this is NULL.
106 scoped_ptr<webrtc::DesktopFrame> output_frame_;
108 // Sub-rectangle of |output_frame_| into which the source will be scaled
109 // and/or letterboxed.
110 webrtc::DesktopRect output_rect_;
112 // Timer used to capture the frame.
113 base::OneShotTimer<Core> capture_timer_;
115 // True when waiting for |desktop_capturer_| to capture current frame.
116 bool capture_in_progress_;
118 // True if the first capture call has returned. Used to log the first capture
120 bool first_capture_returned_;
122 // The type of the capturer.
123 DesktopMediaID::Type capturer_type_;
125 scoped_ptr<webrtc::BasicDesktopFrame> black_frame_;
127 DISALLOW_COPY_AND_ASSIGN(Core);
130 DesktopCaptureDevice::Core::Core(
131 scoped_refptr<base::SingleThreadTaskRunner> task_runner,
132 scoped_ptr<webrtc::DesktopCapturer> capturer,
133 DesktopMediaID::Type type)
134 : task_runner_(task_runner),
135 desktop_capturer_(capturer.Pass()),
136 capture_in_progress_(false),
137 first_capture_returned_(false),
138 capturer_type_(type) {
141 DesktopCaptureDevice::Core::~Core() {
142 DCHECK(task_runner_->BelongsToCurrentThread());
144 output_frame_.reset();
145 previous_frame_size_.set(0, 0);
146 desktop_capturer_.reset();
149 void DesktopCaptureDevice::Core::AllocateAndStart(
150 const media::VideoCaptureParams& params,
151 scoped_ptr<Client> client) {
152 DCHECK(task_runner_->BelongsToCurrentThread());
153 DCHECK_GT(params.requested_format.frame_size.GetArea(), 0);
154 DCHECK_GT(params.requested_format.frame_rate, 0);
155 DCHECK(desktop_capturer_);
156 DCHECK(client.get());
157 DCHECK(!client_.get());
159 client_ = client.Pass();
160 requested_params_ = params;
162 capture_format_ = requested_params_.requested_format;
164 // This capturer always outputs ARGB, non-interlaced.
165 capture_format_.pixel_format = media::PIXEL_FORMAT_ARGB;
167 desktop_capturer_->Start(this);
169 CaptureFrameAndScheduleNext();
172 void DesktopCaptureDevice::Core::SetNotificationWindowId(
173 gfx::NativeViewId window_id) {
174 DCHECK(task_runner_->BelongsToCurrentThread());
176 desktop_capturer_->SetExcludedWindow(window_id);
179 webrtc::SharedMemory*
180 DesktopCaptureDevice::Core::CreateSharedMemory(size_t size) {
184 void DesktopCaptureDevice::Core::OnCaptureCompleted(
185 webrtc::DesktopFrame* frame) {
186 DCHECK(task_runner_->BelongsToCurrentThread());
187 DCHECK(capture_in_progress_);
189 if (!first_capture_returned_) {
190 first_capture_returned_ = true;
191 if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
192 IncrementDesktopCaptureCounter(frame ? FIRST_SCREEN_CAPTURE_SUCCEEDED
193 : FIRST_SCREEN_CAPTURE_FAILED);
195 IncrementDesktopCaptureCounter(frame ? FIRST_WINDOW_CAPTURE_SUCCEEDED
196 : FIRST_WINDOW_CAPTURE_FAILED);
200 capture_in_progress_ = false;
203 std::string log("Failed to capture a frame.");
205 client_->OnError(log);
212 base::TimeDelta capture_time(
213 base::TimeDelta::FromMilliseconds(frame->capture_time_ms()));
215 capturer_type_ == DesktopMediaID::TYPE_SCREEN ? kUmaScreenCaptureTime
216 : kUmaWindowCaptureTime,
219 scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
221 // On OSX We receive a 1x1 frame when the shared window is minimized. It
222 // cannot be subsampled to I420 and will be dropped downstream. So we replace
223 // it with a black frame to avoid the video appearing frozen at the last
225 if (frame->size().width() == 1 || frame->size().height() == 1) {
226 if (!black_frame_.get()) {
228 new webrtc::BasicDesktopFrame(
229 webrtc::DesktopSize(capture_format_.frame_size.width(),
230 capture_format_.frame_size.height())));
231 memset(black_frame_->data(),
233 black_frame_->stride() * black_frame_->size().height());
236 frame = black_frame_.get();
239 // Handle initial frame size and size changes.
240 RefreshCaptureFormat(frame->size());
242 webrtc::DesktopSize output_size(capture_format_.frame_size.width(),
243 capture_format_.frame_size.height());
244 size_t output_bytes = output_size.width() * output_size.height() *
245 webrtc::DesktopFrame::kBytesPerPixel;
246 const uint8_t* output_data = NULL;
247 scoped_ptr<uint8_t[]> flipped_frame_buffer;
249 if (frame->size().equals(output_size)) {
250 // If the captured frame matches the output size, we can return the pixel
251 // data directly, without scaling.
252 output_data = frame->data();
254 // If the |frame| generated by the screen capturer is inverted then we need
256 // This happens only on a specific platform. Refer to crbug.com/306876.
257 if (frame->stride() < 0) {
258 int height = frame->size().height();
260 frame->size().width() * webrtc::DesktopFrame::kBytesPerPixel;
261 flipped_frame_buffer.reset(new uint8_t[output_bytes]);
262 uint8_t* dest = flipped_frame_buffer.get();
263 for (int row = 0; row < height; ++row) {
264 memcpy(dest, output_data, bytes_per_row);
265 dest += bytes_per_row;
266 output_data += frame->stride();
268 output_data = flipped_frame_buffer.get();
271 // Otherwise we need to down-scale and/or letterbox to the target format.
273 // Allocate a buffer of the correct size to scale the frame into.
274 // |output_frame_| is cleared whenever |output_rect_| changes, so we don't
275 // need to worry about clearing out stale pixel data in letterboxed areas.
276 if (!output_frame_) {
277 output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
278 memset(output_frame_->data(), 0, output_bytes);
280 DCHECK(output_frame_->size().equals(output_size));
282 // TODO(wez): Optimize this to scale only changed portions of the output,
283 // using ARGBScaleClip().
284 uint8_t* output_rect_data = output_frame_->data() +
285 output_frame_->stride() * output_rect_.top() +
286 webrtc::DesktopFrame::kBytesPerPixel * output_rect_.left();
287 libyuv::ARGBScale(frame->data(), frame->stride(),
288 frame->size().width(), frame->size().height(),
289 output_rect_data, output_frame_->stride(),
290 output_rect_.width(), output_rect_.height(),
291 libyuv::kFilterBilinear);
292 output_data = output_frame_->data();
295 client_->OnIncomingCapturedData(
296 output_data, output_bytes, capture_format_, 0, base::TimeTicks::Now());
299 void DesktopCaptureDevice::Core::RefreshCaptureFormat(
300 const webrtc::DesktopSize& frame_size) {
301 if (previous_frame_size_.equals(frame_size))
304 // Clear the output frame, if any, since it will either need resizing, or
305 // clearing of stale data in letterbox areas, anyway.
306 output_frame_.reset();
308 if (previous_frame_size_.is_empty() ||
309 requested_params_.allow_resolution_change) {
310 // If this is the first frame, or the receiver supports variable resolution
311 // then determine the output size by treating the requested width & height
313 if (frame_size.width() >
314 requested_params_.requested_format.frame_size.width() ||
315 frame_size.height() >
316 requested_params_.requested_format.frame_size.height()) {
317 output_rect_ = ComputeLetterboxRect(
319 requested_params_.requested_format.frame_size.width(),
320 requested_params_.requested_format.frame_size.height()),
322 output_rect_.Translate(-output_rect_.left(), -output_rect_.top());
324 output_rect_ = webrtc::DesktopRect::MakeSize(frame_size);
326 capture_format_.frame_size.SetSize(output_rect_.width(),
327 output_rect_.height());
329 // Otherwise the output frame size cannot change, so just scale and
331 output_rect_ = ComputeLetterboxRect(
332 webrtc::DesktopSize(capture_format_.frame_size.width(),
333 capture_format_.frame_size.height()),
337 previous_frame_size_ = frame_size;
340 void DesktopCaptureDevice::Core::OnCaptureTimer() {
341 DCHECK(task_runner_->BelongsToCurrentThread());
346 CaptureFrameAndScheduleNext();
349 void DesktopCaptureDevice::Core::CaptureFrameAndScheduleNext() {
350 DCHECK(task_runner_->BelongsToCurrentThread());
352 base::TimeTicks started_time = base::TimeTicks::Now();
354 base::TimeDelta last_capture_duration = base::TimeTicks::Now() - started_time;
356 // Limit frame-rate to reduce CPU consumption.
357 base::TimeDelta capture_period = std::max(
358 (last_capture_duration * 100) / kMaximumCpuConsumptionPercentage,
359 base::TimeDelta::FromSeconds(1) / capture_format_.frame_rate);
361 // Schedule a task for the next frame.
362 capture_timer_.Start(FROM_HERE, capture_period - last_capture_duration,
363 this, &Core::OnCaptureTimer);
366 void DesktopCaptureDevice::Core::DoCapture() {
367 DCHECK(task_runner_->BelongsToCurrentThread());
368 DCHECK(!capture_in_progress_);
370 capture_in_progress_ = true;
371 desktop_capturer_->Capture(webrtc::DesktopRegion());
373 // Currently only synchronous implementations of DesktopCapturer are
375 DCHECK(!capture_in_progress_);
379 scoped_ptr<media::VideoCaptureDevice> DesktopCaptureDevice::Create(
380 const DesktopMediaID& source) {
381 webrtc::DesktopCaptureOptions options =
382 webrtc::DesktopCaptureOptions::CreateDefault();
383 // Leave desktop effects enabled during WebRTC captures.
384 options.set_disable_effects(false);
386 scoped_ptr<webrtc::DesktopCapturer> capturer;
388 switch (source.type) {
389 case DesktopMediaID::TYPE_SCREEN: {
391 options.set_allow_use_magnification_api(true);
393 scoped_ptr<webrtc::ScreenCapturer> screen_capturer(
394 webrtc::ScreenCapturer::Create(options));
395 if (screen_capturer && screen_capturer->SelectScreen(source.id)) {
396 capturer.reset(new webrtc::DesktopAndCursorComposer(
397 screen_capturer.release(),
398 webrtc::MouseCursorMonitor::CreateForScreen(options, source.id)));
399 IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
404 case DesktopMediaID::TYPE_WINDOW: {
405 scoped_ptr<webrtc::WindowCapturer> window_capturer(
406 webrtc::WindowCapturer::Create(options));
407 if (window_capturer && window_capturer->SelectWindow(source.id)) {
408 window_capturer->BringSelectedWindowToFront();
409 capturer.reset(new webrtc::DesktopAndCursorComposer(
410 window_capturer.release(),
411 webrtc::MouseCursorMonitor::CreateForWindow(options, source.id)));
412 IncrementDesktopCaptureCounter(WINDOW_CAPTURER_CREATED);
422 scoped_ptr<media::VideoCaptureDevice> result;
424 result.reset(new DesktopCaptureDevice(capturer.Pass(), source.type));
426 return result.Pass();
429 DesktopCaptureDevice::~DesktopCaptureDevice() {
433 void DesktopCaptureDevice::AllocateAndStart(
434 const media::VideoCaptureParams& params,
435 scoped_ptr<Client> client) {
436 thread_.message_loop_proxy()->PostTask(
438 base::Bind(&Core::AllocateAndStart, base::Unretained(core_.get()), params,
439 base::Passed(&client)));
442 void DesktopCaptureDevice::StopAndDeAllocate() {
444 thread_.message_loop_proxy()->DeleteSoon(FROM_HERE, core_.release());
449 void DesktopCaptureDevice::SetNotificationWindowId(
450 gfx::NativeViewId window_id) {
451 thread_.message_loop_proxy()->PostTask(
453 base::Bind(&Core::SetNotificationWindowId, base::Unretained(core_.get()),
457 DesktopCaptureDevice::DesktopCaptureDevice(
458 scoped_ptr<webrtc::DesktopCapturer> capturer,
459 DesktopMediaID::Type type)
460 : thread_("desktopCaptureThread") {
462 // On Windows the thread must be a UI thread.
463 base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_UI;
465 base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_DEFAULT;
468 thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
470 core_.reset(new Core(thread_.message_loop_proxy(), capturer.Pass(), type));
473 } // namespace content