1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/media/capture/desktop_capture_device_aura.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/timer/timer.h"
10 #include "cc/output/copy_output_request.h"
11 #include "cc/output/copy_output_result.h"
12 #include "content/browser/compositor/image_transport_factory.h"
13 #include "content/browser/media/capture/content_video_capture_device_core.h"
14 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
15 #include "content/common/gpu/client/gl_helper.h"
16 #include "content/public/browser/browser_thread.h"
17 #include "content/public/browser/power_save_blocker.h"
18 #include "media/base/video_util.h"
19 #include "media/video/capture/video_capture_types.h"
20 #include "skia/ext/image_operations.h"
21 #include "third_party/skia/include/core/SkBitmap.h"
22 #include "ui/aura/client/screen_position_client.h"
23 #include "ui/aura/env.h"
24 #include "ui/aura/window.h"
25 #include "ui/aura/window_observer.h"
26 #include "ui/aura/window_tree_host.h"
27 #include "ui/base/cursor/cursors_aura.h"
28 #include "ui/compositor/compositor.h"
29 #include "ui/compositor/dip_util.h"
30 #include "ui/compositor/layer.h"
31 #include "ui/gfx/screen.h"
37 int clip_byte(int x) {
38 return std::max(0, std::min(x, 255));
41 int alpha_blend(int alpha, int src, int dst) {
42 return (src * alpha + dst * (255 - alpha)) / 255;
45 // Helper function to composite a cursor bitmap on a YUV420 video frame.
46 void RenderCursorOnVideoFrame(
47 const scoped_refptr<media::VideoFrame>& target,
48 const SkBitmap& cursor_bitmap,
49 const gfx::Point& cursor_position) {
51 DCHECK(!cursor_bitmap.isNull());
53 gfx::Rect rect = gfx::IntersectRects(
54 gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
55 gfx::Vector2d(cursor_position.x(), cursor_position.y()),
56 target->visible_rect());
58 cursor_bitmap.lockPixels();
59 for (int y = rect.y(); y < rect.bottom(); ++y) {
60 int cursor_y = y - cursor_position.y();
61 uint8* yplane = target->data(media::VideoFrame::kYPlane) +
62 y * target->row_bytes(media::VideoFrame::kYPlane);
63 uint8* uplane = target->data(media::VideoFrame::kUPlane) +
64 (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
65 uint8* vplane = target->data(media::VideoFrame::kVPlane) +
66 (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
67 for (int x = rect.x(); x < rect.right(); ++x) {
68 int cursor_x = x - cursor_position.x();
69 SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
70 int alpha = SkColorGetA(color);
71 int color_r = SkColorGetR(color);
72 int color_g = SkColorGetG(color);
73 int color_b = SkColorGetB(color);
74 int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
76 yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
78 // Only sample U and V at even coordinates.
79 if ((x % 2 == 0) && (y % 2 == 0)) {
80 int color_u = clip_byte(((color_r * -38 + color_g * -74 +
81 color_b * 112 + 128) >> 8) + 128);
82 int color_v = clip_byte(((color_r * 112 + color_g * -94 +
83 color_b * -18 + 128) >> 8) + 128);
84 uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
85 vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
89 cursor_bitmap.unlockPixels();
92 class DesktopVideoCaptureMachine
93 : public VideoCaptureMachine,
94 public aura::WindowObserver,
95 public ui::CompositorObserver,
96 public base::SupportsWeakPtr<DesktopVideoCaptureMachine> {
98 DesktopVideoCaptureMachine(const DesktopMediaID& source);
99 virtual ~DesktopVideoCaptureMachine();
101 // VideoCaptureFrameSource overrides.
102 virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
103 const media::VideoCaptureParams& params) OVERRIDE;
104 virtual void Stop(const base::Closure& callback) OVERRIDE;
106 // Implements aura::WindowObserver.
107 virtual void OnWindowBoundsChanged(aura::Window* window,
108 const gfx::Rect& old_bounds,
109 const gfx::Rect& new_bounds) OVERRIDE;
110 virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE;
111 virtual void OnWindowAddedToRootWindow(aura::Window* window) OVERRIDE;
112 virtual void OnWindowRemovingFromRootWindow(aura::Window* window,
113 aura::Window* new_root) OVERRIDE;
115 // Implements ui::CompositorObserver.
116 virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
117 virtual void OnCompositingStarted(ui::Compositor* compositor,
118 base::TimeTicks start_time) OVERRIDE {}
119 virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
120 virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
121 virtual void OnCompositingLockStateChanged(
122 ui::Compositor* compositor) OVERRIDE {}
126 // |dirty| is false for timer polls and true for compositor updates.
127 void Capture(bool dirty);
129 // Update capture size. Must be called on the UI thread.
130 void UpdateCaptureSize();
132 // Response callback for cc::Layer::RequestCopyOfOutput().
134 scoped_refptr<media::VideoFrame> video_frame,
135 base::TimeTicks start_time,
136 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
137 scoped_ptr<cc::CopyOutputResult> result);
139 // A helper which does the real work for DidCopyOutput. Returns true if
141 bool ProcessCopyOutputResponse(
142 scoped_refptr<media::VideoFrame> video_frame,
143 base::TimeTicks start_time,
144 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
145 scoped_ptr<cc::CopyOutputResult> result);
147 // Helper function to update cursor state.
148 // |region_in_frame| defines the desktop bound in the captured frame.
149 // Returns the current cursor position in captured frame.
150 gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame);
152 // Clears cursor state.
153 void ClearCursorState();
155 // The window associated with the desktop.
156 aura::Window* desktop_window_;
158 // The timer that kicks off period captures.
161 // The id of the window being captured.
162 DesktopMediaID window_id_;
164 // Makes all the decisions about which frames to copy, and how.
165 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
167 // The capture parameters for this capture.
168 media::VideoCaptureParams capture_params_;
170 // YUV readback pipeline.
171 scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_;
174 ui::Cursor last_cursor_;
175 gfx::Point cursor_hot_point_;
176 SkBitmap scaled_cursor_bitmap_;
178 // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
179 // screen from sleeping for the drive-by web.
180 scoped_ptr<PowerSaveBlocker> power_save_blocker_;
182 DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine);
185 DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
186 const DesktopMediaID& source)
187 : desktop_window_(NULL),
189 window_id_(source) {}
191 DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
193 bool DesktopVideoCaptureMachine::Start(
194 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
195 const media::VideoCaptureParams& params) {
196 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
198 desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_);
199 if (!desktop_window_)
202 // If the associated layer is already destroyed then return failure.
203 ui::Layer* layer = desktop_window_->layer();
207 DCHECK(oracle_proxy.get());
208 oracle_proxy_ = oracle_proxy;
209 capture_params_ = params;
211 // Update capture size.
214 // Start observing window events.
215 desktop_window_->AddObserver(this);
217 // Start observing compositor updates.
218 if (desktop_window_->GetHost())
219 desktop_window_->GetHost()->compositor()->AddObserver(this);
221 power_save_blocker_.reset(PowerSaveBlocker::Create(
222 PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
223 "DesktopCaptureDevice is running").release());
226 timer_.Start(FROM_HERE, oracle_proxy_->min_capture_period(),
227 base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(),
233 void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) {
234 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
235 power_save_blocker_.reset();
237 // Stop observing compositor and window events.
238 if (desktop_window_) {
239 if (desktop_window_->GetHost())
240 desktop_window_->GetHost()->compositor()->RemoveObserver(this);
241 desktop_window_->RemoveObserver(this);
242 desktop_window_ = NULL;
251 void DesktopVideoCaptureMachine::UpdateCaptureSize() {
252 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
253 if (oracle_proxy_.get() && desktop_window_) {
254 ui::Layer* layer = desktop_window_->layer();
255 oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
256 layer, layer->bounds().size()));
261 void DesktopVideoCaptureMachine::Capture(bool dirty) {
262 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
264 // Do not capture if the desktop window is already destroyed.
265 if (!desktop_window_)
268 scoped_refptr<media::VideoFrame> frame;
269 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
271 const base::TimeTicks start_time = base::TimeTicks::Now();
272 const VideoCaptureOracle::Event event =
273 dirty ? VideoCaptureOracle::kCompositorUpdate
274 : VideoCaptureOracle::kTimerPoll;
275 if (oracle_proxy_->ObserveEventAndDecideCapture(
276 event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
277 scoped_ptr<cc::CopyOutputRequest> request =
278 cc::CopyOutputRequest::CreateRequest(
279 base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput,
280 AsWeakPtr(), frame, start_time, capture_frame_cb));
281 gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(),
282 desktop_window_->bounds().height());
283 request->set_area(window_rect);
284 desktop_window_->layer()->RequestCopyOfOutput(request.Pass());
288 void CopyOutputFinishedForVideo(
289 base::TimeTicks start_time,
290 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
291 const scoped_refptr<media::VideoFrame>& target,
292 const SkBitmap& cursor_bitmap,
293 const gfx::Point& cursor_position,
294 scoped_ptr<cc::SingleReleaseCallback> release_callback,
296 if (!cursor_bitmap.isNull())
297 RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
298 release_callback->Run(0, false);
299 capture_frame_cb.Run(target, start_time, result);
302 void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
304 cb->Run(sync_point, false);
307 void DesktopVideoCaptureMachine::DidCopyOutput(
308 scoped_refptr<media::VideoFrame> video_frame,
309 base::TimeTicks start_time,
310 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
311 scoped_ptr<cc::CopyOutputResult> result) {
312 static bool first_call = true;
314 bool succeeded = ProcessCopyOutputResponse(
315 video_frame, start_time, capture_frame_cb, result.Pass());
317 base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
319 // The two UMA_ blocks must be put in its own scope since it creates a static
320 // variable which expected constant histogram name.
321 if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
322 UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
324 UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
329 if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
330 IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
331 : FIRST_SCREEN_CAPTURE_FAILED);
333 IncrementDesktopCaptureCounter(succeeded
334 ? FIRST_WINDOW_CAPTURE_SUCCEEDED
335 : FIRST_WINDOW_CAPTURE_FAILED);
340 bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse(
341 scoped_refptr<media::VideoFrame> video_frame,
342 base::TimeTicks start_time,
343 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
344 scoped_ptr<cc::CopyOutputResult> result) {
345 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
346 if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_)
349 if (capture_params_.requested_format.pixel_format ==
350 media::PIXEL_FORMAT_TEXTURE) {
351 DCHECK(!video_frame.get());
352 cc::TextureMailbox texture_mailbox;
353 scoped_ptr<cc::SingleReleaseCallback> release_callback;
354 result->TakeTexture(&texture_mailbox, &release_callback);
355 DCHECK(texture_mailbox.IsTexture());
356 if (!texture_mailbox.IsTexture())
358 video_frame = media::VideoFrame::WrapNativeTexture(
359 make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(),
360 texture_mailbox.target(),
361 texture_mailbox.sync_point())),
362 base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)),
364 gfx::Rect(result->size()),
367 media::VideoFrame::ReadPixelsCB());
368 capture_frame_cb.Run(video_frame, start_time, true);
372 // Compute the dest size we want after the letterboxing resize. Make the
373 // coordinates and sizes even because we letterbox in YUV space
374 // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
375 // line up correctly.
376 // The video frame's coded_size() and the result's size() are both physical
378 gfx::Rect region_in_frame =
379 media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()),
381 region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
382 region_in_frame.y() & ~1,
383 region_in_frame.width() & ~1,
384 region_in_frame.height() & ~1);
385 if (region_in_frame.IsEmpty())
388 ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
389 GLHelper* gl_helper = factory->GetGLHelper();
393 cc::TextureMailbox texture_mailbox;
394 scoped_ptr<cc::SingleReleaseCallback> release_callback;
395 result->TakeTexture(&texture_mailbox, &release_callback);
396 DCHECK(texture_mailbox.IsTexture());
397 if (!texture_mailbox.IsTexture())
400 gfx::Rect result_rect(result->size());
401 if (!yuv_readback_pipeline_ ||
402 yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
403 yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
404 yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
405 yuv_readback_pipeline_.reset(
406 gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
409 video_frame->coded_size(),
415 gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
416 yuv_readback_pipeline_->ReadbackYUV(
417 texture_mailbox.mailbox(),
418 texture_mailbox.sync_point(),
420 base::Bind(&CopyOutputFinishedForVideo,
424 scaled_cursor_bitmap_,
425 cursor_position_in_frame,
426 base::Passed(&release_callback)));
430 gfx::Point DesktopVideoCaptureMachine::UpdateCursorState(
431 const gfx::Rect& region_in_frame) {
432 const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds();
433 gfx::NativeCursor cursor =
434 desktop_window_->GetHost()->last_cursor();
435 if (last_cursor_ != cursor) {
436 SkBitmap cursor_bitmap;
437 if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
438 scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
440 skia::ImageOperations::RESIZE_BEST,
441 cursor_bitmap.width() * region_in_frame.width() /
442 desktop_bounds.width(),
443 cursor_bitmap.height() * region_in_frame.height() /
444 desktop_bounds.height());
445 last_cursor_ = cursor;
447 // Clear cursor state if ui::GetCursorBitmap failed so that we do not
448 // render cursor on the captured frame.
453 gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
454 aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())->
455 ConvertPointFromScreen(desktop_window_, &cursor_position);
456 const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
457 desktop_window_->layer(), cursor_hot_point_);
458 cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
459 -desktop_bounds.y() - hot_point_in_dip.y());
461 region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
462 desktop_bounds.width(),
463 region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
464 desktop_bounds.height());
467 void DesktopVideoCaptureMachine::ClearCursorState() {
468 last_cursor_ = ui::Cursor();
469 cursor_hot_point_ = gfx::Point();
470 scaled_cursor_bitmap_.reset();
473 void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
474 aura::Window* window,
475 const gfx::Rect& old_bounds,
476 const gfx::Rect& new_bounds) {
477 DCHECK(desktop_window_ && window == desktop_window_);
479 // Post task to update capture size on UI thread.
480 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
481 &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
484 void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) {
485 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
487 Stop(base::Bind(&base::DoNothing));
489 oracle_proxy_->ReportError("OnWindowDestroyed()");
492 void DesktopVideoCaptureMachine::OnWindowAddedToRootWindow(
493 aura::Window* window) {
494 DCHECK(window == desktop_window_);
495 window->GetHost()->compositor()->AddObserver(this);
498 void DesktopVideoCaptureMachine::OnWindowRemovingFromRootWindow(
499 aura::Window* window,
500 aura::Window* new_root) {
501 DCHECK(window == desktop_window_);
502 window->GetHost()->compositor()->RemoveObserver(this);
505 void DesktopVideoCaptureMachine::OnCompositingEnded(
506 ui::Compositor* compositor) {
507 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
508 &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true));
513 DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
514 const DesktopMediaID& source)
515 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
516 new DesktopVideoCaptureMachine(source)))) {}
518 DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
519 DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
523 media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
524 const DesktopMediaID& source) {
525 IncrementDesktopCaptureCounter(source.type == DesktopMediaID::TYPE_SCREEN
526 ? SCREEN_CAPTURER_CREATED
527 : WINDOW_CAPTURER_CREATED);
528 return new DesktopCaptureDeviceAura(source);
531 void DesktopCaptureDeviceAura::AllocateAndStart(
532 const media::VideoCaptureParams& params,
533 scoped_ptr<Client> client) {
534 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
535 core_->AllocateAndStart(params, client.Pass());
538 void DesktopCaptureDeviceAura::StopAndDeAllocate() {
539 core_->StopAndDeAllocate();
542 } // namespace content