1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // Implementation notes: This needs to work on a variety of hardware
6 // configurations where the speed of the CPU and GPU greatly affect overall
7 // performance. Spanning several threads, the process of capturing has been
8 // split up into four conceptual stages:
10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's
11 // shared-memory IPC buffer is reserved. There are only a few of these;
12 // when they run out, it indicates that the downstream client -- likely a
13 // video encoder -- is the performance bottleneck, and that the rate of
14 // frame capture should be throttled back.
16 // 2. Capture: A bitmap is snapshotted/copied from the RenderView's backing
17 // store. This is initiated on the UI BrowserThread, and often occurs
18 // asynchronously. Where supported, the GPU scales and color converts
19 // frames to our desired size, and the readback happens directly into the
20 // shared-memory buffer. But this is not always possible, particularly when
21 // accelerated compositing is disabled.
23 // 3. Render (if needed): If the web contents cannot be captured directly into
24 // our target size and color format, scaling and colorspace conversion must
25 // be done on the CPU. A dedicated thread is used for this operation, to
26 // avoid blocking the UI thread. The Render stage always reads from a
27 // bitmap returned by Capture, and writes into the reserved slot in the
28 // shared-memory buffer.
30 // 4. Deliver: The rendered video frame is returned to the client (which
31 // implements the VideoCaptureDevice::Client interface). Because all
32 // paths have written the frame into the IPC buffer, this step should
33 // never need to do an additional copy of the pixel data.
35 // In the best-performing case, the Render step is bypassed: Capture produces
36 // ready-to-Deliver frames. But when accelerated readback is not possible, the
37 // system is designed so that Capture and Render may run concurrently. A timing
38 // diagram helps illustrate this point (@30 FPS):
40 // Time: 0ms 33ms 66ms 99ms
41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt
42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3
44 // In the above example, both capturing and rendering *each* take almost the
45 // full 33 ms available between frames, yet we see that the required throughput
48 // Turning on verbose logging will cause the effective frame rate to be logged
49 // at 5-second intervals.
51 #include "content/browser/renderer_host/media/web_contents_video_capture_device.h"
53 #include "base/basictypes.h"
54 #include "base/bind.h"
55 #include "base/callback_helpers.h"
56 #include "base/logging.h"
57 #include "base/memory/scoped_ptr.h"
58 #include "base/memory/weak_ptr.h"
59 #include "base/message_loop/message_loop_proxy.h"
60 #include "base/metrics/histogram.h"
61 #include "base/sequenced_task_runner.h"
62 #include "base/threading/thread.h"
63 #include "base/threading/thread_checker.h"
64 #include "base/time/time.h"
65 #include "content/browser/renderer_host/media/content_video_capture_device_core.h"
66 #include "content/browser/renderer_host/media/video_capture_oracle.h"
67 #include "content/browser/renderer_host/media/web_contents_capture_util.h"
68 #include "content/browser/renderer_host/render_widget_host_impl.h"
69 #include "content/browser/web_contents/web_contents_impl.h"
70 #include "content/port/browser/render_widget_host_view_frame_subscriber.h"
71 #include "content/port/browser/render_widget_host_view_port.h"
72 #include "content/public/browser/browser_thread.h"
73 #include "content/public/browser/notification_source.h"
74 #include "content/public/browser/notification_types.h"
75 #include "content/public/browser/render_view_host.h"
76 #include "content/public/browser/render_widget_host_view.h"
77 #include "content/public/browser/web_contents_observer.h"
78 #include "media/base/video_util.h"
79 #include "media/video/capture/video_capture_types.h"
80 #include "skia/ext/image_operations.h"
81 #include "third_party/skia/include/core/SkBitmap.h"
82 #include "third_party/skia/include/core/SkColor.h"
88 // Compute a letterbox region, aligned to even coordinates.
89 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size,
90 const gfx::Size& content_size) {
92 gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size),
95 result.set_x(MakeEven(result.x()));
96 result.set_y(MakeEven(result.y()));
97 result.set_width(std::max(kMinFrameWidth, MakeEven(result.width())));
98 result.set_height(std::max(kMinFrameHeight, MakeEven(result.height())));
103 // Wrapper function to invoke ThreadSafeCaptureOracle::CaptureFrameCallback, is
104 // compatible with RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback.
105 void InvokeCaptureFrameCallback(
106 const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
107 base::TimeTicks timestamp,
108 bool frame_captured) {
109 capture_frame_cb.Run(timestamp, frame_captured);
112 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
113 const base::Closure& callback) {
114 render_thread.reset();
116 // After thread join call the callback on UI thread.
117 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
120 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
121 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
122 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
124 FrameSubscriber(VideoCaptureOracle::Event event_type,
125 const scoped_refptr<ThreadSafeCaptureOracle>& oracle)
126 : event_type_(event_type),
127 oracle_proxy_(oracle) {}
129 virtual bool ShouldCaptureFrame(
130 base::TimeTicks present_time,
131 scoped_refptr<media::VideoFrame>* storage,
132 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
133 deliver_frame_cb) OVERRIDE;
136 const VideoCaptureOracle::Event event_type_;
137 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
140 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
141 // whose content is updating, a subscriber that is deciding which of these
142 // updates to capture (and where to deliver them to), and a callback that
143 // knows how to do the capture and prepare the result for delivery.
145 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
146 // the RenderWidgetHostView, to process updates that occur via accelerated
147 // compositing, (b) installing itself as an observer of updates to the
148 // RenderWidgetHost's backing store, to hook updates that occur via software
149 // rendering, and (c) running a timer to possibly initiate non-event-driven
150 // captures that the subscriber might request.
152 // All of this happens on the UI thread, although the
153 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates
154 // autonomously on some other thread.
155 class ContentCaptureSubscription : public content::NotificationObserver {
157 typedef base::Callback<
158 void(const base::TimeTicks&,
159 const scoped_refptr<media::VideoFrame>&,
160 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>
163 // Create a subscription. Whenever a manual capture is required, the
164 // subscription will invoke |capture_callback| on the UI thread to do the
166 ContentCaptureSubscription(
167 const RenderWidgetHost& source,
168 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
169 const CaptureCallback& capture_callback);
170 virtual ~ContentCaptureSubscription();
172 // content::NotificationObserver implementation.
173 virtual void Observe(int type,
174 const content::NotificationSource& source,
175 const content::NotificationDetails& details) OVERRIDE;
180 const int render_process_id_;
181 const int render_view_id_;
183 FrameSubscriber paint_subscriber_;
184 FrameSubscriber timer_subscriber_;
185 content::NotificationRegistrar registrar_;
186 CaptureCallback capture_callback_;
189 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
192 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
193 // invoke |done_cb| to indicate success or failure. |input| is expected to be
194 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
195 // Scaling and letterboxing will be done as needed.
197 // This software implementation should be used only when GPU acceleration of
198 // these activities is not possible. This operation may be expensive (tens to
199 // hundreds of milliseconds), so the caller should ensure that it runs on a
200 // thread where such a pause would cause UI jank.
201 void RenderVideoFrame(const SkBitmap& input,
202 const scoped_refptr<media::VideoFrame>& output,
203 const base::Callback<void(bool)>& done_cb);
205 // Keeps track of the RenderView to be sourced, and executes copying of the
206 // backing store on the UI BrowserThread.
208 // TODO(nick): It would be nice to merge this with WebContentsTracker, but its
209 // implementation is currently asynchronous -- in our case, the "rvh changed"
210 // notification would get posted back to the UI thread and processed later, and
211 // this seems disadvantageous.
212 class WebContentsCaptureMachine
213 : public VideoCaptureMachine,
214 public WebContentsObserver {
216 WebContentsCaptureMachine(int render_process_id, int render_view_id);
217 virtual ~WebContentsCaptureMachine();
219 // VideoCaptureMachine overrides.
221 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) OVERRIDE;
222 virtual void Stop(const base::Closure& callback) OVERRIDE;
224 // Starts a copy from the backing store or the composited surface. Must be run
225 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation
226 // completes. The copy will occur to |target|.
228 // This may be used as a ContentCaptureSubscription::CaptureCallback.
229 void Capture(const base::TimeTicks& start_time,
230 const scoped_refptr<media::VideoFrame>& target,
231 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
234 // content::WebContentsObserver implementation.
235 virtual void DidShowFullscreenWidget(int routing_id) OVERRIDE {
236 fullscreen_widget_id_ = routing_id;
237 RenewFrameSubscription();
240 virtual void DidDestroyFullscreenWidget(int routing_id) OVERRIDE {
241 DCHECK_EQ(fullscreen_widget_id_, routing_id);
242 fullscreen_widget_id_ = MSG_ROUTING_NONE;
243 RenewFrameSubscription();
246 virtual void RenderViewReady() OVERRIDE {
247 RenewFrameSubscription();
250 virtual void AboutToNavigateRenderView(RenderViewHost* rvh) OVERRIDE {
251 RenewFrameSubscription();
254 virtual void DidNavigateMainFrame(
255 const LoadCommittedDetails& details,
256 const FrameNavigateParams& params) OVERRIDE {
257 RenewFrameSubscription();
260 virtual void WebContentsDestroyed(WebContents* web_contents) OVERRIDE;
263 // Starts observing the web contents, returning false if lookup fails.
264 bool StartObservingWebContents();
266 // Helper function to determine the view that we are currently tracking.
267 RenderWidgetHost* GetTarget();
269 // Response callback for RenderWidgetHost::CopyFromBackingStore().
270 void DidCopyFromBackingStore(
271 const base::TimeTicks& start_time,
272 const scoped_refptr<media::VideoFrame>& target,
273 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
276 const SkBitmap& bitmap);
278 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
279 void DidCopyFromCompositingSurfaceToVideoFrame(
280 const base::TimeTicks& start_time,
281 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
285 // Remove the old subscription, and start a new one. This should be called
286 // after any change to the WebContents that affects the RenderWidgetHost or
288 void RenewFrameSubscription();
290 // Parameters saved in constructor.
291 const int initial_render_process_id_;
292 const int initial_render_view_id_;
294 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will
295 // occur. Only used when this activity cannot be done on the GPU.
296 scoped_ptr<base::Thread> render_thread_;
298 // Makes all the decisions about which frames to copy, and how.
299 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
301 // Routing ID of any active fullscreen render widget or MSG_ROUTING_NONE
303 int fullscreen_widget_id_;
305 // Last known RenderView size.
306 gfx::Size last_view_size_;
308 // Responsible for forwarding events from the active RenderWidgetHost to the
309 // oracle, and initiating captures accordingly.
310 scoped_ptr<ContentCaptureSubscription> subscription_;
312 // Weak pointer factory used to invalidate callbacks.
313 base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_;
315 DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine);
318 // Responsible for logging the effective frame rate.
319 // TODO(nick): Make this compatible with the push model and hook it back up.
320 class VideoFrameDeliveryLog {
322 VideoFrameDeliveryLog();
324 // Treat |frame_number| as having been delivered, and update the
325 // frame rate statistics accordingly.
326 void ChronicleFrameDelivery(int frame_number);
329 // The following keep track of and log the effective frame rate whenever
330 // verbose logging is turned on.
331 base::TimeTicks last_frame_rate_log_time_;
332 int count_frames_rendered_;
333 int last_frame_number_;
335 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
338 bool FrameSubscriber::ShouldCaptureFrame(
339 base::TimeTicks present_time,
340 scoped_refptr<media::VideoFrame>* storage,
341 DeliverFrameCallback* deliver_frame_cb) {
342 TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame",
345 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
346 bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
347 event_type_, present_time, storage, &capture_frame_cb);
349 *deliver_frame_cb = base::Bind(&InvokeCaptureFrameCallback, capture_frame_cb);
350 return oracle_decision;
353 ContentCaptureSubscription::ContentCaptureSubscription(
354 const RenderWidgetHost& source,
355 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
356 const CaptureCallback& capture_callback)
357 : render_process_id_(source.GetProcess()->GetID()),
358 render_view_id_(source.GetRoutingID()),
359 paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy),
360 timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy),
361 capture_callback_(capture_callback),
363 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
365 RenderWidgetHostViewPort* view =
366 RenderWidgetHostViewPort::FromRWHV(source.GetView());
368 // Subscribe to accelerated presents. These will be serviced directly by the
370 if (view && kAcceleratedSubscriberIsSupported) {
371 scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
372 new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate,
374 view->BeginFrameSubscription(subscriber.Pass());
377 // Subscribe to software paint events. This instance will service these by
378 // reflecting them back to the WebContentsCaptureMachine via
379 // |capture_callback|.
381 this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE,
382 Source<RenderWidgetHost>(&source));
384 // Subscribe to timer events. This instance will service these as well.
385 timer_.Start(FROM_HERE, oracle_proxy->capture_period(),
386 base::Bind(&ContentCaptureSubscription::OnTimer,
387 base::Unretained(this)));
390 ContentCaptureSubscription::~ContentCaptureSubscription() {
391 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
392 if (kAcceleratedSubscriberIsSupported) {
393 RenderViewHost* source = RenderViewHost::FromID(render_process_id_,
396 RenderWidgetHostViewPort* view =
397 RenderWidgetHostViewPort::FromRWHV(source->GetView());
399 view->EndFrameSubscription();
404 void ContentCaptureSubscription::Observe(
406 const content::NotificationSource& source,
407 const content::NotificationDetails& details) {
408 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
409 DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type);
411 RenderWidgetHostImpl* rwh =
412 RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr());
414 // This message occurs on window resizes and visibility changes even when
415 // accelerated compositing is active, so we need to filter out these cases.
416 if (!rwh || !rwh->GetView() || (rwh->is_accelerated_compositing_active() &&
417 rwh->GetView()->IsSurfaceAvailableForCopy()))
420 TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe",
423 base::Closure copy_done_callback;
424 scoped_refptr<media::VideoFrame> frame;
425 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
426 const base::TimeTicks start_time = base::TimeTicks::Now();
427 if (paint_subscriber_.ShouldCaptureFrame(start_time,
429 &deliver_frame_cb)) {
430 // This message happens just before paint. If we post a task to do the copy,
431 // it should run soon after the paint.
432 BrowserThread::PostTask(
433 BrowserThread::UI, FROM_HERE,
434 base::Bind(capture_callback_, start_time, frame, deliver_frame_cb));
438 void ContentCaptureSubscription::OnTimer() {
439 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
440 TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer");
442 scoped_refptr<media::VideoFrame> frame;
443 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
445 const base::TimeTicks start_time = base::TimeTicks::Now();
446 if (timer_subscriber_.ShouldCaptureFrame(start_time,
448 &deliver_frame_cb)) {
449 capture_callback_.Run(start_time, frame, deliver_frame_cb);
453 void RenderVideoFrame(const SkBitmap& input,
454 const scoped_refptr<media::VideoFrame>& output,
455 const base::Callback<void(bool)>& done_cb) {
456 base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
458 SkAutoLockPixels locker(input);
460 // Sanity-check the captured bitmap.
462 !input.readyToDraw() ||
463 input.config() != SkBitmap::kARGB_8888_Config ||
464 input.width() < 2 || input.height() < 2) {
465 DVLOG(1) << "input unacceptable (size="
467 << ", ready=" << input.readyToDraw()
468 << ", config=" << input.config() << ')';
472 // Sanity-check the output buffer.
473 if (output->format() != media::VideoFrame::I420) {
478 // Calculate the width and height of the content region in the |output|, based
479 // on the aspect ratio of |input|.
480 gfx::Rect region_in_frame = ComputeYV12LetterboxRegion(
481 output->coded_size(), gfx::Size(input.width(), input.height()));
483 // Scale the bitmap to the required size, if necessary.
484 SkBitmap scaled_bitmap;
485 if (input.width() != region_in_frame.width() ||
486 input.height() != region_in_frame.height()) {
488 skia::ImageOperations::ResizeMethod method;
489 if (input.width() < region_in_frame.width() ||
490 input.height() < region_in_frame.height()) {
491 // Avoid box filtering when magnifying, because it's actually
493 method = skia::ImageOperations::RESIZE_HAMMING1;
495 method = skia::ImageOperations::RESIZE_BOX;
498 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale");
499 scaled_bitmap = skia::ImageOperations::Resize(input, method,
500 region_in_frame.width(),
501 region_in_frame.height());
503 scaled_bitmap = input;
506 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV");
508 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
510 media::CopyRGBToVideoFrame(
511 reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
512 scaled_bitmap.rowBytes(),
517 // The result is now ready.
518 ignore_result(failure_handler.Release());
522 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
523 : last_frame_rate_log_time_(),
524 count_frames_rendered_(0),
525 last_frame_number_(0) {
528 void VideoFrameDeliveryLog::ChronicleFrameDelivery(int frame_number) {
529 // Log frame rate, if verbose logging is turned on.
530 static const base::TimeDelta kFrameRateLogInterval =
531 base::TimeDelta::FromSeconds(10);
532 const base::TimeTicks now = base::TimeTicks::Now();
533 if (last_frame_rate_log_time_.is_null()) {
534 last_frame_rate_log_time_ = now;
535 count_frames_rendered_ = 0;
536 last_frame_number_ = frame_number;
538 ++count_frames_rendered_;
539 const base::TimeDelta elapsed = now - last_frame_rate_log_time_;
540 if (elapsed >= kFrameRateLogInterval) {
541 const double measured_fps =
542 count_frames_rendered_ / elapsed.InSecondsF();
543 const int frames_elapsed = frame_number - last_frame_number_;
544 const int count_frames_dropped = frames_elapsed - count_frames_rendered_;
545 DCHECK_LE(0, count_frames_dropped);
546 UMA_HISTOGRAM_PERCENTAGE(
547 "TabCapture.FrameDropPercentage",
548 (count_frames_dropped * 100 + frames_elapsed / 2) / frames_elapsed);
549 UMA_HISTOGRAM_COUNTS(
550 "TabCapture.FrameRate",
551 static_cast<int>(measured_fps));
552 VLOG(1) << "Current measured frame rate for "
553 << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS.";
554 last_frame_rate_log_time_ = now;
555 count_frames_rendered_ = 0;
556 last_frame_number_ = frame_number;
561 WebContentsCaptureMachine::WebContentsCaptureMachine(int render_process_id,
563 : initial_render_process_id_(render_process_id),
564 initial_render_view_id_(render_view_id),
565 fullscreen_widget_id_(MSG_ROUTING_NONE),
566 weak_ptr_factory_(this) {}
568 WebContentsCaptureMachine::~WebContentsCaptureMachine() {
569 BrowserThread::PostBlockingPoolTask(
571 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
572 base::Bind(&base::DoNothing)));
575 bool WebContentsCaptureMachine::Start(
576 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) {
577 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
580 DCHECK(oracle_proxy.get());
581 oracle_proxy_ = oracle_proxy;
583 render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread"));
584 if (!render_thread_->Start()) {
585 DVLOG(1) << "Failed to spawn render thread.";
586 render_thread_.reset();
590 if (!StartObservingWebContents()) {
591 DVLOG(1) << "Failed to observe web contents.";
592 render_thread_.reset();
600 void WebContentsCaptureMachine::Stop(const base::Closure& callback) {
601 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
602 subscription_.reset();
603 if (web_contents()) {
604 web_contents()->DecrementCapturerCount();
608 // Any callback that intend to use render_thread_ will not work after it is
610 weak_ptr_factory_.InvalidateWeakPtrs();
612 // The render thread cannot be stopped on the UI thread, so post a message
613 // to the thread pool used for blocking operations.
614 BrowserThread::PostBlockingPoolTask(
616 base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
622 void WebContentsCaptureMachine::Capture(
623 const base::TimeTicks& start_time,
624 const scoped_refptr<media::VideoFrame>& target,
625 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
627 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
629 RenderWidgetHost* rwh = GetTarget();
630 RenderWidgetHostViewPort* view =
631 rwh ? RenderWidgetHostViewPort::FromRWHV(rwh->GetView()) : NULL;
633 deliver_frame_cb.Run(base::TimeTicks(), false);
637 gfx::Size video_size = target->coded_size();
638 gfx::Size view_size = view->GetViewBounds().size();
639 gfx::Size fitted_size;
640 if (!view_size.IsEmpty()) {
641 fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size();
643 if (view_size != last_view_size_) {
644 last_view_size_ = view_size;
646 // Measure the number of kilopixels.
647 UMA_HISTOGRAM_COUNTS_10000(
648 "TabCapture.ViewChangeKiloPixels",
649 view_size.width() * view_size.height() / 1024);
652 if (!view->IsSurfaceAvailableForCopy()) {
653 // Fallback to the more expensive renderer-side copy if the surface and
654 // backing store are not accessible.
655 rwh->GetSnapshotFromRenderer(
657 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore,
658 weak_ptr_factory_.GetWeakPtr(),
659 start_time, target, deliver_frame_cb));
660 } else if (view->CanCopyToVideoFrame()) {
661 view->CopyFromCompositingSurfaceToVideoFrame(
662 gfx::Rect(view_size),
664 base::Bind(&WebContentsCaptureMachine::
665 DidCopyFromCompositingSurfaceToVideoFrame,
666 weak_ptr_factory_.GetWeakPtr(),
667 start_time, deliver_frame_cb));
669 rwh->CopyFromBackingStore(
671 fitted_size, // Size here is a request not always honored.
672 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore,
673 weak_ptr_factory_.GetWeakPtr(),
674 start_time, target, deliver_frame_cb));
678 bool WebContentsCaptureMachine::StartObservingWebContents() {
679 // Look-up the RenderViewHost and, from that, the WebContents that wraps it.
680 // If successful, begin observing the WebContents instance.
682 // Why this can be unsuccessful: The request for mirroring originates in a
683 // render process, and this request is based on the current RenderView
684 // associated with a tab. However, by the time we get up-and-running here,
685 // there have been multiple back-and-forth IPCs between processes, as well as
686 // a bit of indirection across threads. It's easily possible that, in the
687 // meantime, the original RenderView may have gone away.
688 RenderViewHost* const rvh =
689 RenderViewHost::FromID(initial_render_process_id_,
690 initial_render_view_id_);
691 DVLOG_IF(1, !rvh) << "RenderViewHost::FromID("
692 << initial_render_process_id_ << ", "
693 << initial_render_view_id_ << ") returned NULL.";
694 Observe(rvh ? WebContents::FromRenderViewHost(rvh) : NULL);
696 WebContentsImpl* contents = static_cast<WebContentsImpl*>(web_contents());
698 contents->IncrementCapturerCount(oracle_proxy_->GetCaptureSize());
699 fullscreen_widget_id_ = contents->GetFullscreenWidgetRoutingID();
700 RenewFrameSubscription();
704 DVLOG(1) << "WebContents::FromRenderViewHost(" << rvh << ") returned NULL.";
708 void WebContentsCaptureMachine::WebContentsDestroyed(
709 WebContents* web_contents) {
710 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
712 subscription_.reset();
713 web_contents->DecrementCapturerCount();
714 oracle_proxy_->ReportError("WebContentsDestroyed()");
717 RenderWidgetHost* WebContentsCaptureMachine::GetTarget() {
718 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
722 RenderWidgetHost* rwh = NULL;
723 if (fullscreen_widget_id_ != MSG_ROUTING_NONE) {
724 RenderProcessHost* process = web_contents()->GetRenderProcessHost();
726 rwh = RenderWidgetHost::FromID(process->GetID(), fullscreen_widget_id_);
728 rwh = web_contents()->GetRenderViewHost();
734 void WebContentsCaptureMachine::DidCopyFromBackingStore(
735 const base::TimeTicks& start_time,
736 const scoped_refptr<media::VideoFrame>& target,
737 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
740 const SkBitmap& bitmap) {
741 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
743 base::TimeTicks now = base::TimeTicks::Now();
744 DCHECK(render_thread_.get());
746 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
747 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(),
749 render_thread_->message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
750 &RenderVideoFrame, bitmap, target,
751 base::Bind(deliver_frame_cb, start_time)));
753 // Capture can fail due to transient issues, so just skip this frame.
754 DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
755 deliver_frame_cb.Run(start_time, false);
759 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
760 const base::TimeTicks& start_time,
761 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
764 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
765 base::TimeTicks now = base::TimeTicks::Now();
768 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
770 // Capture can fail due to transient issues, so just skip this frame.
771 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
773 deliver_frame_cb.Run(start_time, success);
776 void WebContentsCaptureMachine::RenewFrameSubscription() {
777 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
779 // Always destroy the old subscription before creating a new one.
780 subscription_.reset();
782 RenderWidgetHost* rwh = GetTarget();
783 if (!rwh || !rwh->GetView())
786 subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_,
787 base::Bind(&WebContentsCaptureMachine::Capture,
788 weak_ptr_factory_.GetWeakPtr())));
793 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice(
794 int render_process_id, int render_view_id)
795 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
796 new WebContentsCaptureMachine(render_process_id, render_view_id)))) {}
798 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() {
799 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying.";
803 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create(
804 const std::string& device_id) {
805 // Parse device_id into render_process_id and render_view_id.
806 int render_process_id = -1;
807 int render_view_id = -1;
808 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget(
809 device_id, &render_process_id, &render_view_id)) {
813 return new WebContentsVideoCaptureDevice(render_process_id, render_view_id);
816 void WebContentsVideoCaptureDevice::AllocateAndStart(
817 const media::VideoCaptureParams& params,
818 scoped_ptr<Client> client) {
819 DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
820 core_->AllocateAndStart(params, client.Pass());
823 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
824 core_->StopAndDeAllocate();
827 } // namespace content