Upstream version 10.38.208.0
[platform/framework/web/crosswalk.git] / src / content / browser / media / capture / desktop_capture_device_aura.cc
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "content/browser/media/capture/desktop_capture_device_aura.h"
6
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/timer/timer.h"
10 #include "cc/output/copy_output_request.h"
11 #include "cc/output/copy_output_result.h"
12 #include "content/browser/compositor/image_transport_factory.h"
13 #include "content/browser/media/capture/content_video_capture_device_core.h"
14 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
15 #include "content/common/gpu/client/gl_helper.h"
16 #include "content/public/browser/browser_thread.h"
17 #include "content/public/browser/power_save_blocker.h"
18 #include "media/base/video_util.h"
19 #include "media/video/capture/video_capture_types.h"
20 #include "skia/ext/image_operations.h"
21 #include "third_party/skia/include/core/SkBitmap.h"
22 #include "ui/aura/client/screen_position_client.h"
23 #include "ui/aura/env.h"
24 #include "ui/aura/window.h"
25 #include "ui/aura/window_observer.h"
26 #include "ui/aura/window_tree_host.h"
27 #include "ui/base/cursor/cursors_aura.h"
28 #include "ui/compositor/compositor.h"
29 #include "ui/compositor/dip_util.h"
30 #include "ui/compositor/layer.h"
31 #include "ui/gfx/screen.h"
32
33 namespace content {
34
35 namespace {
36
37 int clip_byte(int x) {
38   return std::max(0, std::min(x, 255));
39 }
40
41 int alpha_blend(int alpha, int src, int dst) {
42   return (src * alpha + dst * (255 - alpha)) / 255;
43 }
44
45 // Helper function to composite a cursor bitmap on a YUV420 video frame.
46 void RenderCursorOnVideoFrame(
47     const scoped_refptr<media::VideoFrame>& target,
48     const SkBitmap& cursor_bitmap,
49     const gfx::Point& cursor_position) {
50   DCHECK(target);
51   DCHECK(!cursor_bitmap.isNull());
52
53   gfx::Rect rect = gfx::IntersectRects(
54       gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
55           gfx::Vector2d(cursor_position.x(), cursor_position.y()),
56       target->visible_rect());
57
58   cursor_bitmap.lockPixels();
59   for (int y = rect.y(); y < rect.bottom(); ++y) {
60     int cursor_y = y - cursor_position.y();
61     uint8* yplane = target->data(media::VideoFrame::kYPlane) +
62         y * target->row_bytes(media::VideoFrame::kYPlane);
63     uint8* uplane = target->data(media::VideoFrame::kUPlane) +
64         (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
65     uint8* vplane = target->data(media::VideoFrame::kVPlane) +
66         (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
67     for (int x = rect.x(); x < rect.right(); ++x) {
68       int cursor_x = x - cursor_position.x();
69       SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
70       int alpha = SkColorGetA(color);
71       int color_r = SkColorGetR(color);
72       int color_g = SkColorGetG(color);
73       int color_b = SkColorGetB(color);
74       int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
75                                 128) >> 8) + 16);
76       yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
77
78       // Only sample U and V at even coordinates.
79       if ((x % 2 == 0) && (y % 2 == 0)) {
80         int color_u = clip_byte(((color_r * -38 + color_g * -74 +
81                                   color_b * 112 + 128) >> 8) + 128);
82         int color_v = clip_byte(((color_r * 112 + color_g * -94 +
83                                   color_b * -18 + 128) >> 8) + 128);
84         uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
85         vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
86       }
87     }
88   }
89   cursor_bitmap.unlockPixels();
90 }
91
92 class DesktopVideoCaptureMachine
93     : public VideoCaptureMachine,
94       public aura::WindowObserver,
95       public ui::CompositorObserver,
96       public base::SupportsWeakPtr<DesktopVideoCaptureMachine> {
97  public:
98   DesktopVideoCaptureMachine(const DesktopMediaID& source);
99   virtual ~DesktopVideoCaptureMachine();
100
101   // VideoCaptureFrameSource overrides.
102   virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
103                      const media::VideoCaptureParams& params) OVERRIDE;
104   virtual void Stop(const base::Closure& callback) OVERRIDE;
105
106   // Implements aura::WindowObserver.
107   virtual void OnWindowBoundsChanged(aura::Window* window,
108                                      const gfx::Rect& old_bounds,
109                                      const gfx::Rect& new_bounds) OVERRIDE;
110   virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE;
111   virtual void OnWindowAddedToRootWindow(aura::Window* window) OVERRIDE;
112   virtual void OnWindowRemovingFromRootWindow(aura::Window* window,
113                                               aura::Window* new_root) OVERRIDE;
114
115   // Implements ui::CompositorObserver.
116   virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
117   virtual void OnCompositingStarted(ui::Compositor* compositor,
118                                     base::TimeTicks start_time) OVERRIDE {}
119   virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
120   virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
121   virtual void OnCompositingLockStateChanged(
122       ui::Compositor* compositor) OVERRIDE {}
123
124  private:
125   // Captures a frame.
126   // |dirty| is false for timer polls and true for compositor updates.
127   void Capture(bool dirty);
128
129   // Update capture size. Must be called on the UI thread.
130   void UpdateCaptureSize();
131
132   // Response callback for cc::Layer::RequestCopyOfOutput().
133   void DidCopyOutput(
134       scoped_refptr<media::VideoFrame> video_frame,
135       base::TimeTicks start_time,
136       const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
137       scoped_ptr<cc::CopyOutputResult> result);
138
139   // A helper which does the real work for DidCopyOutput. Returns true if
140   // succeeded.
141   bool ProcessCopyOutputResponse(
142       scoped_refptr<media::VideoFrame> video_frame,
143       base::TimeTicks start_time,
144       const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
145       scoped_ptr<cc::CopyOutputResult> result);
146
147   // Helper function to update cursor state.
148   // |region_in_frame| defines the desktop bound in the captured frame.
149   // Returns the current cursor position in captured frame.
150   gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame);
151
152   // Clears cursor state.
153   void ClearCursorState();
154
155   // The window associated with the desktop.
156   aura::Window* desktop_window_;
157
158   // The timer that kicks off period captures.
159   base::Timer timer_;
160
161   // The id of the window being captured.
162   DesktopMediaID window_id_;
163
164   // Makes all the decisions about which frames to copy, and how.
165   scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
166
167   // The capture parameters for this capture.
168   media::VideoCaptureParams capture_params_;
169
170   // YUV readback pipeline.
171   scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_;
172
173   // Cursor state.
174   ui::Cursor last_cursor_;
175   gfx::Point cursor_hot_point_;
176   SkBitmap scaled_cursor_bitmap_;
177
178   // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
179   // screen from sleeping for the drive-by web.
180   scoped_ptr<PowerSaveBlocker> power_save_blocker_;
181
182   DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine);
183 };
184
185 DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
186     const DesktopMediaID& source)
187     : desktop_window_(NULL),
188       timer_(true, true),
189       window_id_(source) {}
190
191 DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
192
193 bool DesktopVideoCaptureMachine::Start(
194     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
195     const media::VideoCaptureParams& params) {
196   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
197
198   desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_);
199   if (!desktop_window_)
200     return false;
201
202   // If the associated layer is already destroyed then return failure.
203   ui::Layer* layer = desktop_window_->layer();
204   if (!layer)
205     return false;
206
207   DCHECK(oracle_proxy.get());
208   oracle_proxy_ = oracle_proxy;
209   capture_params_ = params;
210
211   // Update capture size.
212   UpdateCaptureSize();
213
214   // Start observing window events.
215   desktop_window_->AddObserver(this);
216
217   // Start observing compositor updates.
218   if (desktop_window_->GetHost())
219     desktop_window_->GetHost()->compositor()->AddObserver(this);
220
221   power_save_blocker_.reset(PowerSaveBlocker::Create(
222       PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
223       "DesktopCaptureDevice is running").release());
224
225   // Starts timer.
226   timer_.Start(FROM_HERE, oracle_proxy_->min_capture_period(),
227                base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(),
228                           false));
229
230   started_ = true;
231   return true;
232 }
233
234 void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) {
235   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
236   power_save_blocker_.reset();
237
238   // Stop observing compositor and window events.
239   if (desktop_window_) {
240     if (desktop_window_->GetHost())
241       desktop_window_->GetHost()->compositor()->RemoveObserver(this);
242     desktop_window_->RemoveObserver(this);
243     desktop_window_ = NULL;
244   }
245
246   // Stop timer.
247   timer_.Stop();
248
249   started_ = false;
250
251   callback.Run();
252 }
253
254 void DesktopVideoCaptureMachine::UpdateCaptureSize() {
255   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
256   if (oracle_proxy_ && desktop_window_) {
257     ui::Layer* layer = desktop_window_->layer();
258     gfx::Size capture_size =
259         ui::ConvertSizeToPixel(layer, layer->bounds().size());
260 #if defined(OS_CHROMEOS)
261     // Pad desktop capture size to multiples of 16 pixels to accommodate HW
262     // encoder. TODO(hshi): remove this hack. See http://crbug.com/402151
263     capture_size.SetSize((capture_size.width() + 15) & ~15,
264                          (capture_size.height() + 15) & ~15);
265 #endif
266     oracle_proxy_->UpdateCaptureSize(capture_size);
267   }
268   ClearCursorState();
269 }
270
271 void DesktopVideoCaptureMachine::Capture(bool dirty) {
272   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
273
274   // Do not capture if the desktop window is already destroyed.
275   if (!desktop_window_)
276     return;
277
278   scoped_refptr<media::VideoFrame> frame;
279   ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
280
281   const base::TimeTicks start_time = base::TimeTicks::Now();
282   const VideoCaptureOracle::Event event =
283       dirty ? VideoCaptureOracle::kCompositorUpdate
284             : VideoCaptureOracle::kTimerPoll;
285   if (oracle_proxy_->ObserveEventAndDecideCapture(
286           event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
287     scoped_ptr<cc::CopyOutputRequest> request =
288         cc::CopyOutputRequest::CreateRequest(
289             base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput,
290                        AsWeakPtr(), frame, start_time, capture_frame_cb));
291     gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(),
292                                       desktop_window_->bounds().height());
293     request->set_area(window_rect);
294     desktop_window_->layer()->RequestCopyOfOutput(request.Pass());
295   }
296 }
297
298 void CopyOutputFinishedForVideo(
299     base::TimeTicks start_time,
300     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
301     const scoped_refptr<media::VideoFrame>& target,
302     const SkBitmap& cursor_bitmap,
303     const gfx::Point& cursor_position,
304     scoped_ptr<cc::SingleReleaseCallback> release_callback,
305     bool result) {
306   if (!cursor_bitmap.isNull())
307     RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
308   release_callback->Run(0, false);
309   capture_frame_cb.Run(target, start_time, result);
310 }
311
312 void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
313                               uint32 sync_point) {
314   cb->Run(sync_point, false);
315 }
316
317 void DesktopVideoCaptureMachine::DidCopyOutput(
318     scoped_refptr<media::VideoFrame> video_frame,
319     base::TimeTicks start_time,
320     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
321     scoped_ptr<cc::CopyOutputResult> result) {
322   static bool first_call = true;
323
324   bool succeeded = ProcessCopyOutputResponse(
325       video_frame, start_time, capture_frame_cb, result.Pass());
326
327   base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
328   UMA_HISTOGRAM_TIMES(
329       window_id_.type == DesktopMediaID::TYPE_SCREEN ? kUmaScreenCaptureTime
330                                                      : kUmaWindowCaptureTime,
331       capture_time);
332
333   if (first_call) {
334     first_call = false;
335     if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
336       IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
337                                                : FIRST_SCREEN_CAPTURE_FAILED);
338     } else {
339       IncrementDesktopCaptureCounter(succeeded
340                                          ? FIRST_WINDOW_CAPTURE_SUCCEEDED
341                                          : FIRST_WINDOW_CAPTURE_FAILED);
342     }
343   }
344 }
345
346 bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse(
347     scoped_refptr<media::VideoFrame> video_frame,
348     base::TimeTicks start_time,
349     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
350     scoped_ptr<cc::CopyOutputResult> result) {
351   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
352   if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_)
353     return false;
354
355   if (capture_params_.requested_format.pixel_format ==
356       media::PIXEL_FORMAT_TEXTURE) {
357     DCHECK(!video_frame);
358     cc::TextureMailbox texture_mailbox;
359     scoped_ptr<cc::SingleReleaseCallback> release_callback;
360     result->TakeTexture(&texture_mailbox, &release_callback);
361     DCHECK(texture_mailbox.IsTexture());
362     if (!texture_mailbox.IsTexture())
363       return false;
364     video_frame = media::VideoFrame::WrapNativeTexture(
365         make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(),
366                                                texture_mailbox.target(),
367                                                texture_mailbox.sync_point())),
368         base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)),
369         result->size(),
370         gfx::Rect(result->size()),
371         result->size(),
372         base::TimeDelta(),
373         media::VideoFrame::ReadPixelsCB());
374     capture_frame_cb.Run(video_frame, start_time, true);
375     return true;
376   }
377
378   // Compute the dest size we want after the letterboxing resize. Make the
379   // coordinates and sizes even because we letterbox in YUV space
380   // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
381   // line up correctly.
382   // The video frame's coded_size() and the result's size() are both physical
383   // pixels.
384   gfx::Rect region_in_frame =
385       media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()),
386                                     result->size());
387   region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
388                               region_in_frame.y() & ~1,
389                               region_in_frame.width() & ~1,
390                               region_in_frame.height() & ~1);
391   if (region_in_frame.IsEmpty())
392     return false;
393
394   ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
395   GLHelper* gl_helper = factory->GetGLHelper();
396   if (!gl_helper)
397     return false;
398
399   cc::TextureMailbox texture_mailbox;
400   scoped_ptr<cc::SingleReleaseCallback> release_callback;
401   result->TakeTexture(&texture_mailbox, &release_callback);
402   DCHECK(texture_mailbox.IsTexture());
403   if (!texture_mailbox.IsTexture())
404     return false;
405
406   gfx::Rect result_rect(result->size());
407   if (!yuv_readback_pipeline_ ||
408       yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
409       yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
410       yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
411     yuv_readback_pipeline_.reset(
412         gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
413                                              result_rect.size(),
414                                              result_rect,
415                                              video_frame->coded_size(),
416                                              region_in_frame,
417                                              true,
418                                              true));
419   }
420
421   gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
422   yuv_readback_pipeline_->ReadbackYUV(
423       texture_mailbox.mailbox(),
424       texture_mailbox.sync_point(),
425       video_frame.get(),
426       base::Bind(&CopyOutputFinishedForVideo,
427                  start_time,
428                  capture_frame_cb,
429                  video_frame,
430                  scaled_cursor_bitmap_,
431                  cursor_position_in_frame,
432                  base::Passed(&release_callback)));
433   return true;
434 }
435
436 gfx::Point DesktopVideoCaptureMachine::UpdateCursorState(
437     const gfx::Rect& region_in_frame) {
438   const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds();
439   gfx::NativeCursor cursor =
440       desktop_window_->GetHost()->last_cursor();
441   if (last_cursor_ != cursor) {
442     SkBitmap cursor_bitmap;
443     if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
444       scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
445           cursor_bitmap,
446           skia::ImageOperations::RESIZE_BEST,
447           cursor_bitmap.width() * region_in_frame.width() /
448               desktop_bounds.width(),
449           cursor_bitmap.height() * region_in_frame.height() /
450               desktop_bounds.height());
451       last_cursor_ = cursor;
452     } else {
453       // Clear cursor state if ui::GetCursorBitmap failed so that we do not
454       // render cursor on the captured frame.
455       ClearCursorState();
456     }
457   }
458
459   gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
460   aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())->
461       ConvertPointFromScreen(desktop_window_, &cursor_position);
462   const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
463       desktop_window_->layer(), cursor_hot_point_);
464   cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
465                          -desktop_bounds.y() - hot_point_in_dip.y());
466   return gfx::Point(
467       region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
468           desktop_bounds.width(),
469       region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
470           desktop_bounds.height());
471 }
472
473 void DesktopVideoCaptureMachine::ClearCursorState() {
474   last_cursor_ = ui::Cursor();
475   cursor_hot_point_ = gfx::Point();
476   scaled_cursor_bitmap_.reset();
477 }
478
479 void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
480     aura::Window* window,
481     const gfx::Rect& old_bounds,
482     const gfx::Rect& new_bounds) {
483   DCHECK(desktop_window_ && window == desktop_window_);
484
485   // Post task to update capture size on UI thread.
486   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
487       &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
488 }
489
490 void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) {
491   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
492
493   Stop(base::Bind(&base::DoNothing));
494
495   oracle_proxy_->ReportError("OnWindowDestroyed()");
496 }
497
498 void DesktopVideoCaptureMachine::OnWindowAddedToRootWindow(
499     aura::Window* window) {
500   DCHECK(window == desktop_window_);
501   window->GetHost()->compositor()->AddObserver(this);
502 }
503
504 void DesktopVideoCaptureMachine::OnWindowRemovingFromRootWindow(
505     aura::Window* window,
506     aura::Window* new_root) {
507   DCHECK(window == desktop_window_);
508   window->GetHost()->compositor()->RemoveObserver(this);
509 }
510
511 void DesktopVideoCaptureMachine::OnCompositingEnded(
512     ui::Compositor* compositor) {
513   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
514       &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true));
515 }
516
517 }  // namespace
518
519 DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
520     const DesktopMediaID& source)
521     : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
522         new DesktopVideoCaptureMachine(source)))) {}
523
524 DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
525   DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
526 }
527
528 // static
529 media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
530     const DesktopMediaID& source) {
531   IncrementDesktopCaptureCounter(source.type == DesktopMediaID::TYPE_SCREEN
532                                      ? SCREEN_CAPTURER_CREATED
533                                      : WINDOW_CAPTURER_CREATED);
534   return new DesktopCaptureDeviceAura(source);
535 }
536
537 void DesktopCaptureDeviceAura::AllocateAndStart(
538     const media::VideoCaptureParams& params,
539     scoped_ptr<Client> client) {
540   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
541   core_->AllocateAndStart(params, client.Pass());
542 }
543
544 void DesktopCaptureDeviceAura::StopAndDeAllocate() {
545   core_->StopAndDeAllocate();
546 }
547
548 }  // namespace content