1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // Implementation notes: This needs to work on a variety of hardware
6 // configurations where the speed of the CPU and GPU greatly affect overall
7 // performance. Spanning several threads, the process of capturing has been
8 // split up into four conceptual stages:
10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's
11 // shared-memory IPC buffer is reserved. There are only a few of these;
12 // when they run out, it indicates that the downstream client -- likely a
13 // video encoder -- is the performance bottleneck, and that the rate of
14 // frame capture should be throttled back.
16 // 2. Capture: A bitmap is snapshotted/copied from the RenderView's backing
17 // store. This is initiated on the UI BrowserThread, and often occurs
18 // asynchronously. Where supported, the GPU scales and color converts
19 // frames to our desired size, and the readback happens directly into the
20 // shared-memory buffer. But this is not always possible, particularly when
21 // accelerated compositing is disabled.
23 // 3. Render (if needed): If the web contents cannot be captured directly into
24 // our target size and color format, scaling and colorspace conversion must
25 // be done on the CPU. A dedicated thread is used for this operation, to
26 // avoid blocking the UI thread. The Render stage always reads from a
27 // bitmap returned by Capture, and writes into the reserved slot in the
28 // shared-memory buffer.
30 // 4. Deliver: The rendered video frame is returned to the client (which
31 // implements the VideoCaptureDevice::Client interface). Because all
32 // paths have written the frame into the IPC buffer, this step should
33 // never need to do an additional copy of the pixel data.
35 // In the best-performing case, the Render step is bypassed: Capture produces
36 // ready-to-Deliver frames. But when accelerated readback is not possible, the
37 // system is designed so that Capture and Render may run concurrently. A timing
38 // diagram helps illustrate this point (@30 FPS):
40 // Time: 0ms 33ms 66ms 99ms
41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt
42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3
44 // In the above example, both capturing and rendering *each* take almost the
45 // full 33 ms available between frames, yet we see that the required throughput
48 // Turning on verbose logging will cause the effective frame rate to be logged
49 // at 5-second intervals.
51 #include "content/browser/renderer_host/media/web_contents_video_capture_device.h"
57 #include "base/basictypes.h"
58 #include "base/bind.h"
59 #include "base/callback_forward.h"
60 #include "base/callback_helpers.h"
61 #include "base/debug/trace_event.h"
62 #include "base/logging.h"
63 #include "base/memory/scoped_ptr.h"
64 #include "base/memory/weak_ptr.h"
65 #include "base/message_loop/message_loop_proxy.h"
66 #include "base/metrics/histogram.h"
67 #include "base/sequenced_task_runner.h"
68 #include "base/strings/stringprintf.h"
69 #include "base/synchronization/lock.h"
70 #include "base/threading/thread.h"
71 #include "base/threading/thread_checker.h"
72 #include "base/time/time.h"
73 #include "content/browser/renderer_host/media/video_capture_oracle.h"
74 #include "content/browser/renderer_host/media/web_contents_capture_util.h"
75 #include "content/browser/renderer_host/render_widget_host_impl.h"
76 #include "content/browser/web_contents/web_contents_impl.h"
77 #include "content/port/browser/render_widget_host_view_frame_subscriber.h"
78 #include "content/port/browser/render_widget_host_view_port.h"
79 #include "content/public/browser/browser_thread.h"
80 #include "content/public/browser/notification_source.h"
81 #include "content/public/browser/notification_types.h"
82 #include "content/public/browser/render_process_host.h"
83 #include "content/public/browser/render_view_host.h"
84 #include "content/public/browser/render_widget_host_view.h"
85 #include "content/public/browser/web_contents.h"
86 #include "content/public/browser/web_contents_observer.h"
87 #include "media/base/bind_to_loop.h"
88 #include "media/base/video_frame.h"
89 #include "media/base/video_util.h"
90 #include "media/base/yuv_convert.h"
91 #include "media/video/capture/video_capture_types.h"
92 #include "skia/ext/image_operations.h"
93 #include "third_party/skia/include/core/SkBitmap.h"
94 #include "third_party/skia/include/core/SkColor.h"
95 #include "ui/gfx/rect.h"
96 #include "ui/gfx/skia_util.h"
102 const int kMinFrameWidth = 2;
103 const int kMinFrameHeight = 2;
105 // TODO(nick): Remove this once frame subscription is supported on Aura and
107 #if (defined(OS_WIN) || defined(OS_MACOSX)) || defined(USE_AURA)
108 const bool kAcceleratedSubscriberIsSupported = true;
110 const bool kAcceleratedSubscriberIsSupported = false;
113 // Returns the nearest even integer closer to zero.
114 template<typename IntType>
115 IntType MakeEven(IntType x) {
116 return x & static_cast<IntType>(-2);
119 // Compute a letterbox region, aligned to even coordinates.
120 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size,
121 const gfx::Size& content_size) {
123 gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size),
126 result.set_x(MakeEven(result.x()));
127 result.set_y(MakeEven(result.y()));
128 result.set_width(std::max(kMinFrameWidth, MakeEven(result.width())));
129 result.set_height(std::max(kMinFrameHeight, MakeEven(result.height())));
134 // Thread-safe, refcounted proxy to the VideoCaptureOracle. This proxy wraps
135 // the VideoCaptureOracle, which decides which frames to capture, and a
136 // VideoCaptureDevice::Client, which allocates and receives the captured
137 // frames, in a lock to synchronize state between the two.
138 class ThreadSafeCaptureOracle
139 : public base::RefCountedThreadSafe<ThreadSafeCaptureOracle> {
141 ThreadSafeCaptureOracle(scoped_ptr<media::VideoCaptureDevice::Client> client,
142 scoped_ptr<VideoCaptureOracle> oracle,
143 const gfx::Size& capture_size);
145 bool ObserveEventAndDecideCapture(
146 VideoCaptureOracle::Event event,
147 base::Time event_time,
148 scoped_refptr<media::VideoFrame>* storage,
149 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* callback);
151 base::TimeDelta capture_period() const {
152 return oracle_->capture_period();
155 // Stop new captures from happening (but doesn't forget the client).
158 // Signal an error to the client.
162 friend class base::RefCountedThreadSafe<ThreadSafeCaptureOracle>;
163 virtual ~ThreadSafeCaptureOracle() {}
165 // Callback invoked on completion of all captures.
166 void DidCaptureFrame(const scoped_refptr<media::VideoFrame>& frame,
168 base::Time timestamp,
170 // Protects everything below it.
173 // Recipient of our capture activity.
174 scoped_ptr<media::VideoCaptureDevice::Client> client_;
176 // Makes the decision to capture a frame.
177 const scoped_ptr<VideoCaptureOracle> oracle_;
179 // The resolution at which we're capturing.
180 const gfx::Size capture_size_;
183 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
184 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
185 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
187 FrameSubscriber(VideoCaptureOracle::Event event_type,
188 const scoped_refptr<ThreadSafeCaptureOracle>& oracle)
189 : event_type_(event_type),
190 oracle_proxy_(oracle) {}
192 virtual bool ShouldCaptureFrame(
193 base::Time present_time,
194 scoped_refptr<media::VideoFrame>* storage,
195 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
196 deliver_frame_cb) OVERRIDE;
199 const VideoCaptureOracle::Event event_type_;
200 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
203 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
204 // whose content is updating, a subscriber that is deciding which of these
205 // updates to capture (and where to deliver them to), and a callback that
206 // knows how to do the capture and prepare the result for delivery.
208 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
209 // the RenderWidgetHostView, to process updates that occur via accelerated
210 // compositing, (b) installing itself as an observer of updates to the
211 // RenderWidgetHost's backing store, to hook updates that occur via software
212 // rendering, and (c) running a timer to possibly initiate non-event-driven
213 // captures that the subscriber might request.
215 // All of this happens on the UI thread, although the
216 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates
217 // autonomously on some other thread.
218 class ContentCaptureSubscription : public content::NotificationObserver {
220 typedef base::Callback<void(
222 const scoped_refptr<media::VideoFrame>&,
223 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>
226 // Create a subscription. Whenever a manual capture is required, the
227 // subscription will invoke |capture_callback| on the UI thread to do the
229 ContentCaptureSubscription(
230 const RenderWidgetHost& source,
231 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
232 const CaptureCallback& capture_callback);
233 virtual ~ContentCaptureSubscription();
235 // content::NotificationObserver implementation.
236 virtual void Observe(int type,
237 const content::NotificationSource& source,
238 const content::NotificationDetails& details) OVERRIDE;
243 const int render_process_id_;
244 const int render_view_id_;
246 FrameSubscriber paint_subscriber_;
247 FrameSubscriber timer_subscriber_;
248 content::NotificationRegistrar registrar_;
249 CaptureCallback capture_callback_;
252 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
255 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
256 // invoke |done_cb| to indicate success or failure. |input| is expected to be
257 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
258 // Scaling and letterboxing will be done as needed.
260 // This software implementation should be used only when GPU acceleration of
261 // these activities is not possible. This operation may be expensive (tens to
262 // hundreds of milliseconds), so the caller should ensure that it runs on a
263 // thread where such a pause would cause UI jank.
264 void RenderVideoFrame(const SkBitmap& input,
265 const scoped_refptr<media::VideoFrame>& output,
266 const base::Callback<void(bool)>& done_cb);
268 // Keeps track of the RenderView to be sourced, and executes copying of the
269 // backing store on the UI BrowserThread.
271 // TODO(nick): It would be nice to merge this with WebContentsTracker, but its
272 // implementation is currently asynchronous -- in our case, the "rvh changed"
273 // notification would get posted back to the UI thread and processed later, and
274 // this seems disadvantageous.
275 class CaptureMachine : public WebContentsObserver,
276 public base::SupportsWeakPtr<CaptureMachine> {
278 virtual ~CaptureMachine();
280 // Creates a CaptureMachine. Must be run on the UI BrowserThread. Returns
281 // NULL if the indicated render view cannot be found.
282 static scoped_ptr<CaptureMachine> Create(
283 int render_process_id,
285 const scoped_refptr<base::SequencedTaskRunner>& render_task_runner,
286 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy);
288 // Starts a copy from the backing store or the composited surface. Must be run
289 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation
290 // completes. The copy will occur to |target|.
292 // This may be used as a ContentCaptureSubscription::CaptureCallback.
294 const base::Time& start_time,
295 const scoped_refptr<media::VideoFrame>& target,
296 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
299 // content::WebContentsObserver implementation.
300 virtual void DidShowFullscreenWidget(int routing_id) OVERRIDE {
301 fullscreen_widget_id_ = routing_id;
302 RenewFrameSubscription();
305 virtual void DidDestroyFullscreenWidget(int routing_id) OVERRIDE {
306 DCHECK_EQ(fullscreen_widget_id_, routing_id);
307 fullscreen_widget_id_ = MSG_ROUTING_NONE;
308 RenewFrameSubscription();
311 virtual void RenderViewReady() OVERRIDE {
312 RenewFrameSubscription();
315 virtual void AboutToNavigateRenderView(RenderViewHost* rvh) OVERRIDE {
316 RenewFrameSubscription();
319 virtual void DidNavigateMainFrame(
320 const LoadCommittedDetails& details,
321 const FrameNavigateParams& params) OVERRIDE {
322 RenewFrameSubscription();
325 virtual void WebContentsDestroyed(WebContents* web_contents) OVERRIDE;
329 const scoped_refptr<base::SequencedTaskRunner>& render_task_runner,
330 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy);
332 // Starts observing the web contents, returning false if lookup fails.
333 bool StartObservingWebContents(int initial_render_process_id,
334 int initial_render_view_id);
336 // Helper function to determine the view that we are currently tracking.
337 RenderWidgetHost* GetTarget();
339 // Response callback for RenderWidgetHost::CopyFromBackingStore().
340 void DidCopyFromBackingStore(
341 const base::Time& start_time,
342 const scoped_refptr<media::VideoFrame>& target,
343 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
346 const SkBitmap& bitmap);
348 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
349 void DidCopyFromCompositingSurfaceToVideoFrame(
350 const base::Time& start_time,
351 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
355 // Remove the old subscription, and start a new one. This should be called
356 // after any change to the WebContents that affects the RenderWidgetHost or
358 void RenewFrameSubscription();
360 // The task runner of the thread on which SkBitmap->VideoFrame conversion will
361 // occur. Only used when this activity cannot be done on the GPU.
362 const scoped_refptr<base::SequencedTaskRunner> render_task_runner_;
364 // Makes all the decisions about which frames to copy, and how.
365 const scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
367 // Routing ID of any active fullscreen render widget or MSG_ROUTING_NONE
369 int fullscreen_widget_id_;
371 // Last known RenderView size.
372 gfx::Size last_view_size_;
374 // Responsible for forwarding events from the active RenderWidgetHost to the
375 // oracle, and initiating captures accordingly.
376 scoped_ptr<ContentCaptureSubscription> subscription_;
378 DISALLOW_COPY_AND_ASSIGN(CaptureMachine);
381 // Responsible for logging the effective frame rate.
382 // TODO(nick): Make this compatible with the push model and hook it back up.
383 class VideoFrameDeliveryLog {
385 VideoFrameDeliveryLog();
387 // Treat |frame_number| as having been delivered, and update the
388 // frame rate statistics accordingly.
389 void ChronicleFrameDelivery(int frame_number);
392 // The following keep track of and log the effective frame rate whenever
393 // verbose logging is turned on.
394 base::Time last_frame_rate_log_time_;
395 int count_frames_rendered_;
396 int last_frame_number_;
398 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
401 ThreadSafeCaptureOracle::ThreadSafeCaptureOracle(
402 scoped_ptr<media::VideoCaptureDevice::Client> client,
403 scoped_ptr<VideoCaptureOracle> oracle,
404 const gfx::Size& capture_size)
405 : client_(client.Pass()),
406 oracle_(oracle.Pass()),
407 capture_size_(capture_size) {
410 bool ThreadSafeCaptureOracle::ObserveEventAndDecideCapture(
411 VideoCaptureOracle::Event event,
412 base::Time event_time,
413 scoped_refptr<media::VideoFrame>* storage,
414 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback* callback) {
415 base::AutoLock guard(lock_);
418 return false; // Capture is stopped.
420 scoped_refptr<media::VideoFrame> output_buffer =
421 client_->ReserveOutputBuffer(capture_size_);
422 const bool should_capture =
423 oracle_->ObserveEventAndDecideCapture(event, event_time);
424 const bool content_is_dirty =
425 (event == VideoCaptureOracle::kCompositorUpdate ||
426 event == VideoCaptureOracle::kSoftwarePaint);
427 const char* event_name =
428 (event == VideoCaptureOracle::kTimerPoll ? "poll" :
429 (event == VideoCaptureOracle::kCompositorUpdate ? "gpu" :
432 // Consider the various reasons not to initiate a capture.
433 if (should_capture && !output_buffer.get()) {
434 TRACE_EVENT_INSTANT1("mirroring",
436 TRACE_EVENT_SCOPE_THREAD,
440 } else if (!should_capture && output_buffer.get()) {
441 if (content_is_dirty) {
442 // This is a normal and acceptable way to drop a frame. We've hit our
443 // capture rate limit: for example, the content is animating at 60fps but
444 // we're capturing at 30fps.
445 TRACE_EVENT_INSTANT1("mirroring", "FpsRateLimited",
446 TRACE_EVENT_SCOPE_THREAD,
447 "trigger", event_name);
450 } else if (!should_capture && !output_buffer.get()) {
451 // We decided not to capture, but we wouldn't have been able to if we wanted
452 // to because no output buffer was available.
453 TRACE_EVENT_INSTANT1("mirroring", "NearlyEncodeLimited",
454 TRACE_EVENT_SCOPE_THREAD,
455 "trigger", event_name);
458 int frame_number = oracle_->RecordCapture();
459 TRACE_EVENT_ASYNC_BEGIN2("mirroring", "Capture", output_buffer.get(),
460 "frame_number", frame_number,
461 "trigger", event_name);
462 *storage = output_buffer;
463 *callback = base::Bind(&ThreadSafeCaptureOracle::DidCaptureFrame,
464 this, output_buffer, frame_number);
468 void ThreadSafeCaptureOracle::Stop() {
469 base::AutoLock guard(lock_);
473 void ThreadSafeCaptureOracle::ReportError() {
474 base::AutoLock guard(lock_);
479 void ThreadSafeCaptureOracle::DidCaptureFrame(
480 const scoped_refptr<media::VideoFrame>& frame,
482 base::Time timestamp,
484 base::AutoLock guard(lock_);
485 TRACE_EVENT_ASYNC_END2("mirroring", "Capture", frame.get(),
487 "timestamp", timestamp.ToInternalValue());
490 return; // Capture is stopped.
493 if (oracle_->CompleteCapture(frame_number, timestamp))
494 client_->OnIncomingCapturedVideoFrame(frame, timestamp);
498 bool FrameSubscriber::ShouldCaptureFrame(
499 base::Time present_time,
500 scoped_refptr<media::VideoFrame>* storage,
501 DeliverFrameCallback* deliver_frame_cb) {
502 TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame",
505 return oracle_proxy_->ObserveEventAndDecideCapture(event_type_, present_time,
506 storage, deliver_frame_cb);
509 ContentCaptureSubscription::ContentCaptureSubscription(
510 const RenderWidgetHost& source,
511 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
512 const CaptureCallback& capture_callback)
513 : render_process_id_(source.GetProcess()->GetID()),
514 render_view_id_(source.GetRoutingID()),
515 paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy),
516 timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy),
517 capture_callback_(capture_callback),
519 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
521 RenderWidgetHostViewPort* view =
522 RenderWidgetHostViewPort::FromRWHV(source.GetView());
524 // Subscribe to accelerated presents. These will be serviced directly by the
526 if (view && kAcceleratedSubscriberIsSupported) {
527 scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
528 new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate,
530 view->BeginFrameSubscription(subscriber.Pass());
533 // Subscribe to software paint events. This instance will service these by
534 // reflecting them back to the CaptureMachine via |capture_callback|.
536 this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE,
537 Source<RenderWidgetHost>(&source));
539 // Subscribe to timer events. This instance will service these as well.
540 timer_.Start(FROM_HERE, oracle_proxy->capture_period(),
541 base::Bind(&ContentCaptureSubscription::OnTimer,
542 base::Unretained(this)));
545 ContentCaptureSubscription::~ContentCaptureSubscription() {
546 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
547 if (kAcceleratedSubscriberIsSupported) {
548 RenderViewHost* source = RenderViewHost::FromID(render_process_id_,
551 RenderWidgetHostViewPort* view =
552 RenderWidgetHostViewPort::FromRWHV(source->GetView());
554 view->EndFrameSubscription();
559 void ContentCaptureSubscription::Observe(
561 const content::NotificationSource& source,
562 const content::NotificationDetails& details) {
563 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
564 DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type);
566 RenderWidgetHostImpl* rwh =
567 RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr());
569 // This message occurs on window resizes and visibility changes even when
570 // accelerated compositing is active, so we need to filter out these cases.
571 if (!rwh || !rwh->GetView() || (rwh->is_accelerated_compositing_active() &&
572 rwh->GetView()->IsSurfaceAvailableForCopy()))
575 TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe",
578 base::Closure copy_done_callback;
579 scoped_refptr<media::VideoFrame> frame;
580 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
581 const base::Time start_time = base::Time::Now();
582 if (paint_subscriber_.ShouldCaptureFrame(start_time,
584 &deliver_frame_cb)) {
585 // This message happens just before paint. If we post a task to do the copy,
586 // it should run soon after the paint.
587 BrowserThread::PostTask(
588 BrowserThread::UI, FROM_HERE,
589 base::Bind(capture_callback_, start_time, frame, deliver_frame_cb));
593 void ContentCaptureSubscription::OnTimer() {
594 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
595 TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer");
597 scoped_refptr<media::VideoFrame> frame;
598 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
600 const base::Time start_time = base::Time::Now();
601 if (timer_subscriber_.ShouldCaptureFrame(start_time,
603 &deliver_frame_cb)) {
604 capture_callback_.Run(start_time, frame, deliver_frame_cb);
608 void RenderVideoFrame(const SkBitmap& input,
609 const scoped_refptr<media::VideoFrame>& output,
610 const base::Callback<void(bool)>& done_cb) {
611 base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
613 SkAutoLockPixels locker(input);
615 // Sanity-check the captured bitmap.
617 !input.readyToDraw() ||
618 input.config() != SkBitmap::kARGB_8888_Config ||
619 input.width() < 2 || input.height() < 2) {
620 DVLOG(1) << "input unacceptable (size="
622 << ", ready=" << input.readyToDraw()
623 << ", config=" << input.config() << ')';
627 // Sanity-check the output buffer.
628 if (output->format() != media::VideoFrame::I420) {
633 // Calculate the width and height of the content region in the |output|, based
634 // on the aspect ratio of |input|.
635 gfx::Rect region_in_frame = ComputeYV12LetterboxRegion(
636 output->coded_size(), gfx::Size(input.width(), input.height()));
638 // Scale the bitmap to the required size, if necessary.
639 SkBitmap scaled_bitmap;
640 if (input.width() != region_in_frame.width() ||
641 input.height() != region_in_frame.height()) {
643 skia::ImageOperations::ResizeMethod method;
644 if (input.width() < region_in_frame.width() ||
645 input.height() < region_in_frame.height()) {
646 // Avoid box filtering when magnifying, because it's actually
648 method = skia::ImageOperations::RESIZE_HAMMING1;
650 method = skia::ImageOperations::RESIZE_BOX;
653 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale");
654 scaled_bitmap = skia::ImageOperations::Resize(input, method,
655 region_in_frame.width(),
656 region_in_frame.height());
658 scaled_bitmap = input;
661 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV");
663 SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
665 media::CopyRGBToVideoFrame(
666 reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
667 scaled_bitmap.rowBytes(),
672 // The result is now ready.
673 ignore_result(failure_handler.Release());
677 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
678 : last_frame_rate_log_time_(),
679 count_frames_rendered_(0),
680 last_frame_number_(0) {
683 void VideoFrameDeliveryLog::ChronicleFrameDelivery(int frame_number) {
684 // Log frame rate, if verbose logging is turned on.
685 static const base::TimeDelta kFrameRateLogInterval =
686 base::TimeDelta::FromSeconds(10);
687 const base::Time now = base::Time::Now();
688 if (last_frame_rate_log_time_.is_null()) {
689 last_frame_rate_log_time_ = now;
690 count_frames_rendered_ = 0;
691 last_frame_number_ = frame_number;
693 ++count_frames_rendered_;
694 const base::TimeDelta elapsed = now - last_frame_rate_log_time_;
695 if (elapsed >= kFrameRateLogInterval) {
696 const double measured_fps =
697 count_frames_rendered_ / elapsed.InSecondsF();
698 const int frames_elapsed = frame_number - last_frame_number_;
699 const int count_frames_dropped = frames_elapsed - count_frames_rendered_;
700 DCHECK_LE(0, count_frames_dropped);
701 UMA_HISTOGRAM_PERCENTAGE(
702 "TabCapture.FrameDropPercentage",
703 (count_frames_dropped * 100 + frames_elapsed / 2) / frames_elapsed);
704 UMA_HISTOGRAM_COUNTS(
705 "TabCapture.FrameRate",
706 static_cast<int>(measured_fps));
707 VLOG(1) << "Current measured frame rate for "
708 << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS.";
709 last_frame_rate_log_time_ = now;
710 count_frames_rendered_ = 0;
711 last_frame_number_ = frame_number;
717 scoped_ptr<CaptureMachine> CaptureMachine::Create(
718 int render_process_id,
720 const scoped_refptr<base::SequencedTaskRunner>& render_task_runner,
721 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) {
722 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
723 DCHECK(render_task_runner.get());
724 DCHECK(oracle_proxy.get());
725 scoped_ptr<CaptureMachine> machine(
726 new CaptureMachine(render_task_runner, oracle_proxy));
728 if (!machine->StartObservingWebContents(render_process_id, render_view_id))
731 return machine.Pass();
734 CaptureMachine::CaptureMachine(
735 const scoped_refptr<base::SequencedTaskRunner>& render_task_runner,
736 const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy)
737 : render_task_runner_(render_task_runner),
738 oracle_proxy_(oracle_proxy),
739 fullscreen_widget_id_(MSG_ROUTING_NONE) {}
741 CaptureMachine::~CaptureMachine() {
742 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) ||
743 !BrowserThread::IsMessageLoopValid(BrowserThread::UI));
745 // Stop observing the web contents.
746 subscription_.reset();
747 if (web_contents()) {
748 web_contents()->DecrementCapturerCount();
753 void CaptureMachine::Capture(
754 const base::Time& start_time,
755 const scoped_refptr<media::VideoFrame>& target,
756 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
758 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
760 RenderWidgetHost* rwh = GetTarget();
761 RenderWidgetHostViewPort* view =
762 rwh ? RenderWidgetHostViewPort::FromRWHV(rwh->GetView()) : NULL;
764 deliver_frame_cb.Run(base::Time(), false);
768 gfx::Size video_size = target->coded_size();
769 gfx::Size view_size = view->GetViewBounds().size();
770 gfx::Size fitted_size;
771 if (!view_size.IsEmpty()) {
772 fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size();
774 if (view_size != last_view_size_) {
775 last_view_size_ = view_size;
777 // Measure the number of kilopixels.
778 UMA_HISTOGRAM_COUNTS_10000(
779 "TabCapture.ViewChangeKiloPixels",
780 view_size.width() * view_size.height() / 1024);
783 if (!view->IsSurfaceAvailableForCopy()) {
784 // Fallback to the more expensive renderer-side copy if the surface and
785 // backing store are not accessible.
786 rwh->GetSnapshotFromRenderer(
788 base::Bind(&CaptureMachine::DidCopyFromBackingStore, this->AsWeakPtr(),
789 start_time, target, deliver_frame_cb));
790 } else if (view->CanCopyToVideoFrame()) {
791 view->CopyFromCompositingSurfaceToVideoFrame(
792 gfx::Rect(view_size),
794 base::Bind(&CaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame,
795 this->AsWeakPtr(), start_time, deliver_frame_cb));
797 rwh->CopyFromBackingStore(
799 fitted_size, // Size here is a request not always honored.
800 base::Bind(&CaptureMachine::DidCopyFromBackingStore, this->AsWeakPtr(),
801 start_time, target, deliver_frame_cb));
805 bool CaptureMachine::StartObservingWebContents(int initial_render_process_id,
806 int initial_render_view_id) {
807 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
809 // Look-up the RenderViewHost and, from that, the WebContents that wraps it.
810 // If successful, begin observing the WebContents instance.
812 // Why this can be unsuccessful: The request for mirroring originates in a
813 // render process, and this request is based on the current RenderView
814 // associated with a tab. However, by the time we get up-and-running here,
815 // there have been multiple back-and-forth IPCs between processes, as well as
816 // a bit of indirection across threads. It's easily possible that, in the
817 // meantime, the original RenderView may have gone away.
818 RenderViewHost* const rvh =
819 RenderViewHost::FromID(initial_render_process_id,
820 initial_render_view_id);
821 DVLOG_IF(1, !rvh) << "RenderViewHost::FromID("
822 << initial_render_process_id << ", "
823 << initial_render_view_id << ") returned NULL.";
824 Observe(rvh ? WebContents::FromRenderViewHost(rvh) : NULL);
826 WebContentsImpl* contents = static_cast<WebContentsImpl*>(web_contents());
828 contents->IncrementCapturerCount();
829 fullscreen_widget_id_ = contents->GetFullscreenWidgetRoutingID();
830 RenewFrameSubscription();
834 DVLOG(1) << "WebContents::FromRenderViewHost(" << rvh << ") returned NULL.";
838 void CaptureMachine::WebContentsDestroyed(WebContents* web_contents) {
839 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
841 subscription_.reset();
842 web_contents->DecrementCapturerCount();
843 oracle_proxy_->ReportError();
846 RenderWidgetHost* CaptureMachine::GetTarget() {
847 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
851 RenderWidgetHost* rwh = NULL;
852 if (fullscreen_widget_id_ != MSG_ROUTING_NONE) {
853 RenderProcessHost* process = web_contents()->GetRenderProcessHost();
855 rwh = RenderWidgetHost::FromID(process->GetID(), fullscreen_widget_id_);
857 rwh = web_contents()->GetRenderViewHost();
863 void CaptureMachine::DidCopyFromBackingStore(
864 const base::Time& start_time,
865 const scoped_refptr<media::VideoFrame>& target,
866 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
869 const SkBitmap& bitmap) {
870 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
872 base::Time now = base::Time::Now();
874 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
875 TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(),
877 render_task_runner_->PostTask(FROM_HERE, base::Bind(
878 &RenderVideoFrame, bitmap, target,
879 base::Bind(deliver_frame_cb, start_time)));
881 // Capture can fail due to transient issues, so just skip this frame.
882 DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
883 deliver_frame_cb.Run(start_time, false);
887 void CaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
888 const base::Time& start_time,
889 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
892 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
893 base::Time now = base::Time::Now();
896 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
898 // Capture can fail due to transient issues, so just skip this frame.
899 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
901 deliver_frame_cb.Run(start_time, success);
904 void CaptureMachine::RenewFrameSubscription() {
905 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
907 // Always destroy the old subscription before creating a new one.
908 subscription_.reset();
910 RenderWidgetHost* rwh = GetTarget();
911 if (!rwh || !rwh->GetView())
914 subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_,
915 base::Bind(&CaptureMachine::Capture, this->AsWeakPtr())));
918 void DeleteCaptureMachineOnUIThread(
919 scoped_ptr<CaptureMachine> capture_machine) {
920 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
921 capture_machine.reset();
926 // The "meat" of the video capture implementation, which is a ref-counted class.
927 // Separating this from the "shell class" WebContentsVideoCaptureDevice allows
928 // safe destruction without needing to block any threads (e.g., the IO
931 // WebContentsVideoCaptureDevice::Impl manages a simple state machine and the
932 // pipeline (see notes at top of this file). It times the start of successive
933 // captures and facilitates the processing of each through the stages of the
935 class WebContentsVideoCaptureDevice::Impl : public base::SupportsWeakPtr<Impl> {
938 Impl(int render_process_id, int render_view_id);
941 // Asynchronous requests to change WebContentsVideoCaptureDevice::Impl state.
942 void AllocateAndStart(int width,
945 scoped_ptr<media::VideoCaptureDevice::Client> client);
946 void StopAndDeAllocate();
950 // Flag indicating current state.
957 void TransitionStateTo(State next_state);
959 // Stops capturing and notifies client_ of an error state.
962 // Called in response to CaptureMachine::Create that runs on the UI thread.
963 // It will assign the capture machine to the Impl class if it still exists
964 // otherwise it will post a task to delete CaptureMachine on the UI thread.
965 static void AssignCaptureMachine(
966 base::WeakPtr<WebContentsVideoCaptureDevice::Impl> impl,
967 scoped_ptr<CaptureMachine> capture_machine);
969 // Tracks that all activity occurs on the media stream manager's thread.
970 base::ThreadChecker thread_checker_;
972 // These values identify the starting view that will be captured. After
973 // capture starts, the target view IDs will change as navigation occurs, and
974 // so these values are not relevant after the initial bootstrapping.
975 const int initial_render_process_id_;
976 const int initial_render_view_id_;
978 // Current lifecycle state.
981 // A dedicated worker thread for doing image operations. Started/joined here,
982 // but used by the CaptureMachine.
983 base::Thread render_thread_;
985 // Tracks the CaptureMachine that's doing work on our behalf on the UI thread.
986 // This value should never be dereferenced by this class, other than to
987 // create and destroy it on the UI thread.
988 scoped_ptr<CaptureMachine> capture_machine_;
990 // Our thread-safe capture oracle which serves as the gateway to the video
991 // capture pipeline. Besides the WCVCD itself, it is the only component of the
992 // system with direct access to |client_|.
993 scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
995 DISALLOW_COPY_AND_ASSIGN(Impl);
998 WebContentsVideoCaptureDevice::Impl::Impl(int render_process_id,
1000 : initial_render_process_id_(render_process_id),
1001 initial_render_view_id_(render_view_id),
1003 render_thread_("WebContentsVideo_RenderThread") {}
1005 void WebContentsVideoCaptureDevice::Impl::AllocateAndStart(
1009 scoped_ptr<VideoCaptureDevice::Client> client) {
1010 DCHECK(thread_checker_.CalledOnValidThread());
1012 if (state_ != kIdle) {
1013 DVLOG(1) << "Allocate() invoked when not in state Idle.";
1017 if (frame_rate <= 0) {
1018 DVLOG(1) << "invalid frame_rate: " << frame_rate;
1023 if (!render_thread_.Start()) {
1024 DVLOG(1) << "Failed to spawn render thread.";
1029 // Frame dimensions must each be a positive, even integer, since the client
1030 // wants (or will convert to) YUV420.
1031 width = MakeEven(width);
1032 height = MakeEven(height);
1033 if (width < kMinFrameWidth || height < kMinFrameHeight) {
1034 DVLOG(1) << "invalid width (" << width << ") and/or height ("
1040 // Need to call OnFrameInfo just to set the frame rate.
1041 // TODO(nick): http://crbug.com/309907 The other parameters of this struct are
1042 // ignored. Come up with a better way to communicate the frame rate.
1043 media::VideoCaptureCapability settings;
1044 settings.frame_rate = frame_rate;
1045 client->OnFrameInfo(settings);
1047 base::TimeDelta capture_period = base::TimeDelta::FromMicroseconds(
1048 1000000.0 / frame_rate + 0.5);
1050 scoped_ptr<VideoCaptureOracle> oracle(
1051 new VideoCaptureOracle(capture_period,
1052 kAcceleratedSubscriberIsSupported));
1053 oracle_proxy_ = new ThreadSafeCaptureOracle(
1054 client.Pass(), oracle.Pass(), gfx::Size(width, height));
1056 // Allocates the CaptureMachine. The CaptureMachine will be tracking render
1057 // view swapping over its lifetime, and we don't want to lose our reference to
1058 // the current render view by starting over with the stale
1059 // |initial_render_view_id_|.
1060 DCHECK(!capture_machine_.get());
1061 BrowserThread::PostTaskAndReplyWithResult(
1062 BrowserThread::UI, FROM_HERE,
1063 base::Bind(&CaptureMachine::Create,
1064 initial_render_process_id_,
1065 initial_render_view_id_,
1066 render_thread_.message_loop_proxy(), oracle_proxy_),
1067 base::Bind(&Impl::AssignCaptureMachine, AsWeakPtr()));
1069 TransitionStateTo(kCapturing);
1073 void WebContentsVideoCaptureDevice::Impl::AssignCaptureMachine(
1074 base::WeakPtr<WebContentsVideoCaptureDevice::Impl> impl,
1075 scoped_ptr<CaptureMachine> capture_machine) {
1076 DCHECK(!impl.get() || impl->thread_checker_.CalledOnValidThread());
1079 // If WCVD::Impl was destroyed before we got back on it's thread and
1080 // capture_machine is not NULL, then we need to return to the UI thread to
1081 // safely cleanup the CaptureMachine.
1082 if (capture_machine) {
1083 BrowserThread::PostTask(
1084 BrowserThread::UI, FROM_HERE, base::Bind(
1085 &DeleteCaptureMachineOnUIThread, base::Passed(&capture_machine)));
1088 } else if (!capture_machine) {
1091 impl->capture_machine_ = capture_machine.Pass();
1095 void WebContentsVideoCaptureDevice::Impl::StopAndDeAllocate() {
1096 DCHECK(thread_checker_.CalledOnValidThread());
1098 if (state_ != kCapturing) {
1101 oracle_proxy_->Stop();
1102 oracle_proxy_ = NULL;
1103 render_thread_.Stop();
1105 TransitionStateTo(kIdle);
1107 // There is still a capture pipeline running that is checking in with the
1108 // oracle, and processing captures that are already started in flight. That
1109 // pipeline must be shut down asynchronously, on the UI thread.
1110 if (capture_machine_) {
1111 // The task that is posted to the UI thread might not run if we are shutting
1112 // down, so we transfer ownership of CaptureMachine to the closure so that
1113 // it is still cleaned up when the closure is deleted.
1114 BrowserThread::PostTask(
1115 BrowserThread::UI, FROM_HERE, base::Bind(
1116 &DeleteCaptureMachineOnUIThread, base::Passed(&capture_machine_)));
1120 WebContentsVideoCaptureDevice::Impl::~Impl() {
1121 DCHECK(!capture_machine_) << "Cleanup on UI thread did not happen.";
1122 DVLOG(1) << "WebContentsVideoCaptureDevice::Impl@" << this << " destroying.";
1125 void WebContentsVideoCaptureDevice::Impl::TransitionStateTo(State next_state) {
1126 DCHECK(thread_checker_.CalledOnValidThread());
1129 static const char* kStateNames[] = {
1130 "Idle", "Allocated", "Capturing", "Error"
1132 DVLOG(1) << "State change: " << kStateNames[state_]
1133 << " --> " << kStateNames[next_state];
1136 state_ = next_state;
1139 void WebContentsVideoCaptureDevice::Impl::Error() {
1140 DCHECK(thread_checker_.CalledOnValidThread());
1142 if (state_ == kIdle)
1146 oracle_proxy_->ReportError();
1148 StopAndDeAllocate();
1149 TransitionStateTo(kError);
1152 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice(
1153 int render_process_id,
1155 : impl_(new WebContentsVideoCaptureDevice::Impl(render_process_id,
1158 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() {
1159 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying.";
1163 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create(
1164 const std::string& device_id) {
1165 // Parse device_id into render_process_id and render_view_id.
1166 int render_process_id = -1;
1167 int render_view_id = -1;
1168 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget(
1169 device_id, &render_process_id, &render_view_id)) {
1173 return new WebContentsVideoCaptureDevice(render_process_id, render_view_id);
1176 void WebContentsVideoCaptureDevice::AllocateAndStart(
1177 const media::VideoCaptureCapability& capture_format,
1178 scoped_ptr<Client> client) {
1179 DVLOG(1) << "Allocating " << capture_format.width << "x"
1180 << capture_format.height;
1181 impl_->AllocateAndStart(capture_format.width,
1182 capture_format.height,
1183 capture_format.frame_rate,
1187 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
1188 impl_->StopAndDeAllocate();
1191 } // namespace content