src/content/browser/media/capture/desktop_capture_device_aura.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device_aura.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/timer/timer.h"
  10 #include "cc/output/copy_output_request.h"
  11 #include "cc/output/copy_output_result.h"
  12 #include "content/browser/compositor/image_transport_factory.h"
  13 #include "content/browser/media/capture/content_video_capture_device_core.h"
  14 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  15 #include "content/common/gpu/client/gl_helper.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "media/base/bind_to_current_loop.h"
  18 #include "media/base/video_util.h"
  19 #include "media/video/capture/video_capture_types.h"
  20 #include "skia/ext/image_operations.h"
  21 #include "third_party/skia/include/core/SkBitmap.h"
  22 #include "ui/aura/env.h"
  23 #include "ui/aura/window.h"
  24 #include "ui/aura/window_observer.h"
  25 #include "ui/aura/window_tree_host.h"
  26 #include "ui/base/cursor/cursors_aura.h"
  27 #include "ui/compositor/compositor.h"
  28 #include "ui/compositor/dip_util.h"
  29 #include "ui/compositor/layer.h"
  30 #include "ui/gfx/screen.h"
  31
  32 namespace content {
  33
  34 namespace {
  35
  36 int clip_byte(int x) {
  37   return std::max(0, std::min(x, 255));
  38 }
  39
  40 int alpha_blend(int alpha, int src, int dst) {
  41   return (src * alpha + dst * (255 - alpha)) / 255;
  42 }
  43
  44 // Helper function to composite a cursor bitmap on a YUV420 video frame.
  45 void RenderCursorOnVideoFrame(
  46     const scoped_refptr<media::VideoFrame>& target,
  47     const SkBitmap& cursor_bitmap,
  48     const gfx::Point& cursor_position) {
  49   DCHECK(target);
  50   DCHECK(!cursor_bitmap.isNull());
  51
  52   gfx::Rect rect = gfx::IntersectRects(
  53       gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
  54           gfx::Vector2d(cursor_position.x(), cursor_position.y()),
  55       target->visible_rect());
  56
  57   cursor_bitmap.lockPixels();
  58   for (int y = rect.y(); y < rect.bottom(); ++y) {
  59     int cursor_y = y - cursor_position.y();
  60     uint8* yplane = target->data(media::VideoFrame::kYPlane) +
  61         y * target->row_bytes(media::VideoFrame::kYPlane);
  62     uint8* uplane = target->data(media::VideoFrame::kUPlane) +
  63         (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
  64     uint8* vplane = target->data(media::VideoFrame::kVPlane) +
  65         (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
  66     for (int x = rect.x(); x < rect.right(); ++x) {
  67       int cursor_x = x - cursor_position.x();
  68       SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
  69       int alpha = SkColorGetA(color);
  70       int color_r = SkColorGetR(color);
  71       int color_g = SkColorGetG(color);
  72       int color_b = SkColorGetB(color);
  73       int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
  74                                 128) >> 8) + 16);
  75       yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
  76
  77       // Only sample U and V at even coordinates.
  78       if ((x % 2 == 0) && (y % 2 == 0)) {
  79         int color_u = clip_byte(((color_r * -38 + color_g * -74 +
  80                                   color_b * 112 + 128) >> 8) + 128);
  81         int color_v = clip_byte(((color_r * 112 + color_g * -94 +
  82                                   color_b * -18 + 128) >> 8) + 128);
  83         uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
  84         vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
  85       }
  86     }
  87   }
  88   cursor_bitmap.unlockPixels();
  89 }
  90
  91 class DesktopVideoCaptureMachine
  92     : public VideoCaptureMachine,
  93       public aura::WindowObserver,
  94       public ui::CompositorObserver,
  95       public base::SupportsWeakPtr<DesktopVideoCaptureMachine> {
  96  public:
  97   DesktopVideoCaptureMachine(const DesktopMediaID& source);
  98   virtual ~DesktopVideoCaptureMachine();
  99
 100   // VideoCaptureFrameSource overrides.
 101   virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 102                      const media::VideoCaptureParams& params) OVERRIDE;
 103   virtual void Stop(const base::Closure& callback) OVERRIDE;
 104
 105   // Implements aura::WindowObserver.
 106   virtual void OnWindowBoundsChanged(aura::Window* window,
 107                                      const gfx::Rect& old_bounds,
 108                                      const gfx::Rect& new_bounds) OVERRIDE;
 109   virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE;
 110
 111   // Implements ui::CompositorObserver.
 112   virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
 113   virtual void OnCompositingStarted(ui::Compositor* compositor,
 114                                     base::TimeTicks start_time) OVERRIDE {}
 115   virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
 116   virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
 117   virtual void OnCompositingLockStateChanged(
 118       ui::Compositor* compositor) OVERRIDE {}
 119
 120  private:
 121   // Captures a frame.
 122   // |dirty| is false for timer polls and true for compositor updates.
 123   void Capture(bool dirty);
 124
 125   // Update capture size. Must be called on the UI thread.
 126   void UpdateCaptureSize();
 127
 128   // Response callback for cc::Layer::RequestCopyOfOutput().
 129   void DidCopyOutput(
 130       scoped_refptr<media::VideoFrame> video_frame,
 131       base::TimeTicks start_time,
 132       const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 133       scoped_ptr<cc::CopyOutputResult> result);
 134
 135   // A helper which does the real work for DidCopyOutput. Returns true if
 136   // succeeded.
 137   bool ProcessCopyOutputResponse(
 138       scoped_refptr<media::VideoFrame> video_frame,
 139       base::TimeTicks start_time,
 140       const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 141       scoped_ptr<cc::CopyOutputResult> result);
 142
 143   // Helper function to update cursor state.
 144   // |region_in_frame| defines the desktop bound in the captured frame.
 145   // Returns the current cursor position in captured frame.
 146   gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame);
 147
 148   // Clears cursor state.
 149   void ClearCursorState();
 150
 151   // The window associated with the desktop.
 152   aura::Window* desktop_window_;
 153
 154   // The layer associated with the desktop.
 155   ui::Layer* desktop_layer_;
 156
 157   // The timer that kicks off period captures.
 158   base::Timer timer_;
 159
 160   // The id of the window being captured.
 161   DesktopMediaID window_id_;
 162
 163   // Makes all the decisions about which frames to copy, and how.
 164   scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
 165
 166   // The capture parameters for this capture.
 167   media::VideoCaptureParams capture_params_;
 168
 169   // YUV readback pipeline.
 170   scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_;
 171
 172   // Cursor state.
 173   ui::Cursor last_cursor_;
 174   gfx::Point cursor_hot_point_;
 175   SkBitmap scaled_cursor_bitmap_;
 176
 177   DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine);
 178 };
 179
 180 DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
 181     const DesktopMediaID& source)
 182     : desktop_window_(NULL),
 183       desktop_layer_(NULL),
 184       timer_(true, true),
 185       window_id_(source) {}
 186
 187 DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
 188
 189 bool DesktopVideoCaptureMachine::Start(
 190     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 191     const media::VideoCaptureParams& params) {
 192   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 193
 194   desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_);
 195   if (!desktop_window_)
 196     return false;
 197
 198   // If the desktop layer is already destroyed then return failure.
 199   desktop_layer_ = desktop_window_->layer();
 200   if (!desktop_layer_)
 201     return false;
 202
 203   DCHECK(oracle_proxy.get());
 204   oracle_proxy_ = oracle_proxy;
 205   capture_params_ = params;
 206
 207   // Update capture size.
 208   UpdateCaptureSize();
 209
 210   // Start observing window events.
 211   desktop_window_->AddObserver(this);
 212
 213   // Start observing compositor updates.
 214   ui::Compositor* compositor = desktop_layer_->GetCompositor();
 215   if (!compositor)
 216     return false;
 217
 218   compositor->AddObserver(this);
 219
 220   // Starts timer.
 221   timer_.Start(FROM_HERE, oracle_proxy_->capture_period(),
 222                base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(),
 223                           false));
 224
 225   started_ = true;
 226   return true;
 227 }
 228
 229 void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) {
 230   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 231
 232   // Stop observing window events.
 233   if (desktop_window_) {
 234     desktop_window_->RemoveObserver(this);
 235     desktop_window_ = NULL;
 236   }
 237
 238   // Stop observing compositor updates.
 239   if (desktop_layer_) {
 240     ui::Compositor* compositor = desktop_layer_->GetCompositor();
 241     if (compositor)
 242       compositor->RemoveObserver(this);
 243     desktop_layer_ = NULL;
 244   }
 245
 246   // Stop timer.
 247   timer_.Stop();
 248
 249   started_ = false;
 250
 251   callback.Run();
 252 }
 253
 254 void DesktopVideoCaptureMachine::UpdateCaptureSize() {
 255   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 256   if (oracle_proxy_ && desktop_layer_) {
 257     oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
 258         desktop_layer_, desktop_layer_->bounds().size()));
 259   }
 260   ClearCursorState();
 261 }
 262
 263 void DesktopVideoCaptureMachine::Capture(bool dirty) {
 264   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 265
 266   // Do not capture if the desktop layer is already destroyed.
 267   if (!desktop_layer_)
 268     return;
 269
 270   scoped_refptr<media::VideoFrame> frame;
 271   ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 272
 273   const base::TimeTicks start_time = base::TimeTicks::Now();
 274   const VideoCaptureOracle::Event event =
 275       dirty ? VideoCaptureOracle::kCompositorUpdate
 276             : VideoCaptureOracle::kTimerPoll;
 277   if (oracle_proxy_->ObserveEventAndDecideCapture(
 278       event, start_time, &frame, &capture_frame_cb)) {
 279     scoped_ptr<cc::CopyOutputRequest> request =
 280         cc::CopyOutputRequest::CreateRequest(
 281             base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput,
 282                        AsWeakPtr(), frame, start_time, capture_frame_cb));
 283     gfx::Rect window_rect =
 284         ui::ConvertRectToPixel(desktop_window_->layer(),
 285                                gfx::Rect(desktop_window_->bounds().width(),
 286                                          desktop_window_->bounds().height()));
 287     request->set_area(window_rect);
 288     desktop_layer_->RequestCopyOfOutput(request.Pass());
 289   }
 290 }
 291
 292 void CopyOutputFinishedForVideo(
 293     base::TimeTicks start_time,
 294     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 295     const scoped_refptr<media::VideoFrame>& target,
 296     const SkBitmap& cursor_bitmap,
 297     const gfx::Point& cursor_position,
 298     scoped_ptr<cc::SingleReleaseCallback> release_callback,
 299     bool result) {
 300   if (!cursor_bitmap.isNull())
 301     RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
 302   release_callback->Run(0, false);
 303   capture_frame_cb.Run(target, start_time, result);
 304 }
 305
 306 void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
 307                               const std::vector<uint32>& sync_points) {
 308   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 309   GLHelper* gl_helper = ImageTransportFactory::GetInstance()->GetGLHelper();
 310   DCHECK(gl_helper);
 311   for (unsigned i = 0; i < sync_points.size(); i++)
 312     gl_helper->WaitSyncPoint(sync_points[i]);
 313   uint32 new_sync_point = gl_helper->InsertSyncPoint();
 314   cb->Run(new_sync_point, false);
 315 }
 316
 317 void DesktopVideoCaptureMachine::DidCopyOutput(
 318     scoped_refptr<media::VideoFrame> video_frame,
 319     base::TimeTicks start_time,
 320     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 321     scoped_ptr<cc::CopyOutputResult> result) {
 322   static bool first_call = true;
 323
 324   bool succeeded = ProcessCopyOutputResponse(
 325       video_frame, start_time, capture_frame_cb, result.Pass());
 326
 327   base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
 328   UMA_HISTOGRAM_TIMES(
 329       window_id_.type == DesktopMediaID::TYPE_SCREEN ? kUmaScreenCaptureTime
 330                                                      : kUmaWindowCaptureTime,
 331       capture_time);
 332
 333   if (first_call) {
 334     first_call = false;
 335     if (window_id_.type == DesktopMediaID::TYPE_SCREEN) {
 336       IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 337                                                : FIRST_SCREEN_CAPTURE_FAILED);
 338     } else {
 339       IncrementDesktopCaptureCounter(succeeded
 340                                          ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 341                                          : FIRST_WINDOW_CAPTURE_SUCCEEDED);
 342     }
 343   }
 344 }
 345
 346 bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse(
 347     scoped_refptr<media::VideoFrame> video_frame,
 348     base::TimeTicks start_time,
 349     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 350     scoped_ptr<cc::CopyOutputResult> result) {
 351   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 352   if (result->IsEmpty() || result->size().IsEmpty() || !desktop_layer_)
 353     return false;
 354
 355   if (capture_params_.requested_format.pixel_format ==
 356       media::PIXEL_FORMAT_TEXTURE) {
 357     DCHECK(!video_frame);
 358     cc::TextureMailbox texture_mailbox;
 359     scoped_ptr<cc::SingleReleaseCallback> release_callback;
 360     result->TakeTexture(&texture_mailbox, &release_callback);
 361     DCHECK(texture_mailbox.IsTexture());
 362     if (!texture_mailbox.IsTexture())
 363       return false;
 364     video_frame = media::VideoFrame::WrapNativeTexture(
 365         make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(),
 366                                                texture_mailbox.target(),
 367                                                texture_mailbox.sync_point())),
 368         media::BindToCurrentLoop(base::Bind(&RunSingleReleaseCallback,
 369                                             base::Passed(&release_callback))),
 370         result->size(),
 371         gfx::Rect(result->size()),
 372         result->size(),
 373         base::TimeDelta(),
 374         media::VideoFrame::ReadPixelsCB());
 375     capture_frame_cb.Run(video_frame, start_time, true);
 376     return true;
 377   }
 378
 379   // Compute the dest size we want after the letterboxing resize. Make the
 380   // coordinates and sizes even because we letterbox in YUV space
 381   // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
 382   // line up correctly.
 383   // The video frame's coded_size() and the result's size() are both physical
 384   // pixels.
 385   gfx::Rect region_in_frame =
 386       media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()),
 387                                     result->size());
 388   region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
 389                               region_in_frame.y() & ~1,
 390                               region_in_frame.width() & ~1,
 391                               region_in_frame.height() & ~1);
 392   if (region_in_frame.IsEmpty())
 393     return false;
 394
 395   ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
 396   GLHelper* gl_helper = factory->GetGLHelper();
 397   if (!gl_helper)
 398     return false;
 399
 400   cc::TextureMailbox texture_mailbox;
 401   scoped_ptr<cc::SingleReleaseCallback> release_callback;
 402   result->TakeTexture(&texture_mailbox, &release_callback);
 403   DCHECK(texture_mailbox.IsTexture());
 404   if (!texture_mailbox.IsTexture())
 405     return false;
 406
 407   gfx::Rect result_rect(result->size());
 408   if (!yuv_readback_pipeline_ ||
 409       yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
 410       yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
 411       yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
 412     yuv_readback_pipeline_.reset(
 413         gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
 414                                              result_rect.size(),
 415                                              result_rect,
 416                                              video_frame->coded_size(),
 417                                              region_in_frame,
 418                                              true,
 419                                              true));
 420   }
 421
 422   gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
 423   yuv_readback_pipeline_->ReadbackYUV(
 424       texture_mailbox.mailbox(),
 425       texture_mailbox.sync_point(),
 426       video_frame.get(),
 427       base::Bind(&CopyOutputFinishedForVideo,
 428                  start_time,
 429                  capture_frame_cb,
 430                  video_frame,
 431                  scaled_cursor_bitmap_,
 432                  cursor_position_in_frame,
 433                  base::Passed(&release_callback)));
 434   return true;
 435 }
 436
 437 gfx::Point DesktopVideoCaptureMachine::UpdateCursorState(
 438     const gfx::Rect& region_in_frame) {
 439   const gfx::Rect desktop_bounds = desktop_layer_->bounds();
 440   gfx::NativeCursor cursor =
 441       desktop_window_->GetHost()->last_cursor();
 442   if (last_cursor_ != cursor) {
 443     SkBitmap cursor_bitmap;
 444     if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
 445       scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
 446           cursor_bitmap,
 447           skia::ImageOperations::RESIZE_BEST,
 448           cursor_bitmap.width() * region_in_frame.width() /
 449               desktop_bounds.width(),
 450           cursor_bitmap.height() * region_in_frame.height() /
 451               desktop_bounds.height());
 452       last_cursor_ = cursor;
 453     } else {
 454       // Clear cursor state if ui::GetCursorBitmap failed so that we do not
 455       // render cursor on the captured frame.
 456       ClearCursorState();
 457     }
 458   }
 459
 460   gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
 461   const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
 462       desktop_layer_, cursor_hot_point_);
 463   cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
 464                          -desktop_bounds.y() - hot_point_in_dip.y());
 465   return gfx::Point(
 466       region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
 467           desktop_bounds.width(),
 468       region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
 469           desktop_bounds.height());
 470 }
 471
 472 void DesktopVideoCaptureMachine::ClearCursorState() {
 473   last_cursor_ = ui::Cursor();
 474   cursor_hot_point_ = gfx::Point();
 475   scaled_cursor_bitmap_.reset();
 476 }
 477
 478 void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
 479     aura::Window* window,
 480     const gfx::Rect& old_bounds,
 481     const gfx::Rect& new_bounds) {
 482   DCHECK(desktop_window_ && window == desktop_window_);
 483
 484   // Post task to update capture size on UI thread.
 485   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 486       &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
 487 }
 488
 489 void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) {
 490   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 491
 492   Stop(base::Bind(&base::DoNothing));
 493
 494   oracle_proxy_->ReportError("OnWindowDestroyed()");
 495 }
 496
 497 void DesktopVideoCaptureMachine::OnCompositingEnded(
 498     ui::Compositor* compositor) {
 499   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 500       &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true));
 501 }
 502
 503 }  // namespace
 504
 505 DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
 506     const DesktopMediaID& source)
 507     : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
 508         new DesktopVideoCaptureMachine(source)))) {}
 509
 510 DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
 511   DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
 512 }
 513
 514 // static
 515 media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
 516     const DesktopMediaID& source) {
 517   IncrementDesktopCaptureCounter(source.type == DesktopMediaID::TYPE_SCREEN
 518                                      ? SCREEN_CAPTURER_CREATED
 519                                      : WINDOW_CATPTURER_CREATED);
 520   return new DesktopCaptureDeviceAura(source);
 521 }
 522
 523 void DesktopCaptureDeviceAura::AllocateAndStart(
 524     const media::VideoCaptureParams& params,
 525     scoped_ptr<Client> client) {
 526   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
 527   core_->AllocateAndStart(params, client.Pass());
 528 }
 529
 530 void DesktopCaptureDeviceAura::StopAndDeAllocate() {
 531   core_->StopAndDeAllocate();
 532 }
 533
 534 }  // namespace content