1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/renderer_host/media/video_capture_controller.h"
10 #include "base/bind.h"
11 #include "base/debug/trace_event.h"
12 #include "base/metrics/histogram.h"
13 #include "base/metrics/sparse_histogram.h"
14 #include "base/stl_util.h"
15 #include "content/browser/renderer_host/media/media_stream_manager.h"
16 #include "content/browser/renderer_host/media/video_capture_manager.h"
17 #include "content/public/browser/browser_thread.h"
18 #include "gpu/command_buffer/common/mailbox_holder.h"
19 #include "media/base/video_frame.h"
20 #include "media/base/video_util.h"
21 #include "media/base/yuv_convert.h"
23 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
24 #include "third_party/libyuv/include/libyuv.h"
27 using media::VideoCaptureFormat;
33 static const int kInfiniteRatio = 99999;
35 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
36 UMA_HISTOGRAM_SPARSE_SLOWLY( \
38 (height) ? ((width) * 100) / (height) : kInfiniteRatio);
40 // The number of buffers that VideoCaptureBufferPool should allocate.
41 const int kNoOfBuffers = 3;
43 class PoolBuffer : public media::VideoCaptureDevice::Client::Buffer {
45 PoolBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
49 : Buffer(buffer_id, data, size), pool_(pool) {
54 virtual ~PoolBuffer() { pool_->RelinquishProducerReservation(id()); }
56 const scoped_refptr<VideoCaptureBufferPool> pool_;
59 } // anonymous namespace
61 struct VideoCaptureController::ControllerClient {
62 ControllerClient(const VideoCaptureControllerID& id,
63 VideoCaptureControllerEventHandler* handler,
64 base::ProcessHandle render_process,
65 media::VideoCaptureSessionId session_id,
66 const media::VideoCaptureParams& params)
68 event_handler(handler),
69 render_process_handle(render_process),
70 session_id(session_id),
72 session_closed(false) {}
74 ~ControllerClient() {}
76 // ID used for identifying this object.
77 const VideoCaptureControllerID controller_id;
78 VideoCaptureControllerEventHandler* const event_handler;
80 // Handle to the render process that will receive the capture buffers.
81 const base::ProcessHandle render_process_handle;
82 const media::VideoCaptureSessionId session_id;
83 const media::VideoCaptureParams parameters;
85 // Buffers that are currently known to this client.
86 std::set<int> known_buffers;
88 // Buffers currently held by this client, and syncpoint callback to call when
89 // they are returned from the client.
90 typedef std::map<int, scoped_refptr<media::VideoFrame> > ActiveBufferMap;
91 ActiveBufferMap active_buffers;
93 // State of capture session, controlled by VideoCaptureManager directly. This
94 // transitions to true as soon as StopSession() occurs, at which point the
95 // client is sent an OnEnded() event. However, because the client retains a
96 // VideoCaptureController* pointer, its ControllerClient entry lives on until
97 // it unregisters itself via RemoveClient(), which may happen asynchronously.
99 // TODO(nick): If we changed the semantics of VideoCaptureHost so that
100 // OnEnded() events were processed synchronously (with the RemoveClient() done
101 // implicitly), we could avoid tracking this state here in the Controller, and
102 // simplify the code in both places.
106 // Receives events from the VideoCaptureDevice and posts them to a
107 // VideoCaptureController on the IO thread. An instance of this class may safely
108 // outlive its target VideoCaptureController.
110 // Methods of this class may be called from any thread, and in practice will
111 // often be called on some auxiliary thread depending on the platform and the
112 // device type; including, for example, the DirectShow thread on Windows, the
113 // v4l2_thread on Linux, and the UI thread for tab capture.
114 class VideoCaptureController::VideoCaptureDeviceClient
115 : public media::VideoCaptureDevice::Client {
117 explicit VideoCaptureDeviceClient(
118 const base::WeakPtr<VideoCaptureController>& controller,
119 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
120 virtual ~VideoCaptureDeviceClient();
122 // VideoCaptureDevice::Client implementation.
123 virtual scoped_refptr<Buffer> ReserveOutputBuffer(
124 media::VideoFrame::Format format,
125 const gfx::Size& size) OVERRIDE;
126 virtual void OnIncomingCapturedData(const uint8* data,
128 const VideoCaptureFormat& frame_format,
130 base::TimeTicks timestamp) OVERRIDE;
131 virtual void OnIncomingCapturedVideoFrame(
132 const scoped_refptr<Buffer>& buffer,
133 const VideoCaptureFormat& buffer_format,
134 const scoped_refptr<media::VideoFrame>& frame,
135 base::TimeTicks timestamp) OVERRIDE;
136 virtual void OnError(const std::string& reason) OVERRIDE;
139 scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format,
140 const gfx::Size& dimensions);
142 // The controller to which we post events.
143 const base::WeakPtr<VideoCaptureController> controller_;
145 // The pool of shared-memory buffers used for capturing.
146 const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
151 VideoCaptureController::VideoCaptureController()
152 : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)),
153 state_(VIDEO_CAPTURE_STATE_STARTED),
154 weak_ptr_factory_(this) {
157 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
158 const base::WeakPtr<VideoCaptureController>& controller,
159 const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
160 : controller_(controller), buffer_pool_(buffer_pool), first_frame_(true) {}
162 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
164 base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() {
165 return weak_ptr_factory_.GetWeakPtr();
168 scoped_ptr<media::VideoCaptureDevice::Client>
169 VideoCaptureController::NewDeviceClient() {
170 scoped_ptr<media::VideoCaptureDevice::Client> result(
171 new VideoCaptureDeviceClient(this->GetWeakPtr(), buffer_pool_));
172 return result.Pass();
175 void VideoCaptureController::AddClient(
176 const VideoCaptureControllerID& id,
177 VideoCaptureControllerEventHandler* event_handler,
178 base::ProcessHandle render_process,
179 media::VideoCaptureSessionId session_id,
180 const media::VideoCaptureParams& params) {
181 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
182 DVLOG(1) << "VideoCaptureController::AddClient, id " << id.device_id
183 << ", " << params.requested_format.frame_size.ToString()
184 << ", " << params.requested_format.frame_rate
185 << ", " << session_id
188 // If this is the first client added to the controller, cache the parameters.
189 if (!controller_clients_.size())
190 video_capture_format_ = params.requested_format;
192 // Signal error in case device is already in error state.
193 if (state_ == VIDEO_CAPTURE_STATE_ERROR) {
194 event_handler->OnError(id);
198 // Do nothing if this client has called AddClient before.
199 if (FindClient(id, event_handler, controller_clients_))
202 ControllerClient* client = new ControllerClient(
203 id, event_handler, render_process, session_id, params);
204 // If we already have gotten frame_info from the device, repeat it to the new
206 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
207 controller_clients_.push_back(client);
212 int VideoCaptureController::RemoveClient(
213 const VideoCaptureControllerID& id,
214 VideoCaptureControllerEventHandler* event_handler) {
215 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
216 DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id.device_id;
218 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
220 return kInvalidMediaCaptureSessionId;
222 // Take back all buffers held by the |client|.
223 for (ControllerClient::ActiveBufferMap::iterator buffer_it =
224 client->active_buffers.begin();
225 buffer_it != client->active_buffers.end();
227 buffer_pool_->RelinquishConsumerHold(buffer_it->first, 1);
229 client->active_buffers.clear();
231 int session_id = client->session_id;
232 controller_clients_.remove(client);
238 void VideoCaptureController::StopSession(int session_id) {
239 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
240 DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
242 ControllerClient* client = FindClient(session_id, controller_clients_);
245 client->session_closed = true;
246 client->event_handler->OnEnded(client->controller_id);
250 void VideoCaptureController::ReturnBuffer(
251 const VideoCaptureControllerID& id,
252 VideoCaptureControllerEventHandler* event_handler,
255 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
257 ControllerClient* client = FindClient(id, event_handler, controller_clients_);
259 // If this buffer is not held by this client, or this client doesn't exist
260 // in controller, do nothing.
261 ControllerClient::ActiveBufferMap::iterator iter;
262 if (!client || (iter = client->active_buffers.find(buffer_id)) ==
263 client->active_buffers.end()) {
267 scoped_refptr<media::VideoFrame> frame = iter->second;
268 client->active_buffers.erase(iter);
270 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE)
271 frame->mailbox_holder()->sync_point = sync_point;
273 buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
276 const media::VideoCaptureFormat&
277 VideoCaptureController::GetVideoCaptureFormat() const {
278 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
279 return video_capture_format_;
282 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
283 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
284 media::VideoFrame::Format format,
285 const gfx::Size& size) {
286 return DoReserveOutputBuffer(format, size);
289 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedData(
292 const VideoCaptureFormat& frame_format,
294 base::TimeTicks timestamp) {
295 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
297 if (!frame_format.IsValid())
300 // Chopped pixels in width/height in case video capture device has odd
301 // numbers for width/height.
302 int chopped_width = 0;
303 int chopped_height = 0;
304 int new_unrotated_width = frame_format.frame_size.width();
305 int new_unrotated_height = frame_format.frame_size.height();
307 if (new_unrotated_width & 1) {
308 --new_unrotated_width;
311 if (new_unrotated_height & 1) {
312 --new_unrotated_height;
316 int destination_width = new_unrotated_width;
317 int destination_height = new_unrotated_height;
318 if (rotation == 90 || rotation == 270) {
319 destination_width = new_unrotated_height;
320 destination_height = new_unrotated_width;
322 const gfx::Size dimensions(destination_width, destination_height);
323 if (!media::VideoFrame::IsValidConfig(media::VideoFrame::I420,
325 gfx::Rect(dimensions),
330 scoped_refptr<Buffer> buffer =
331 DoReserveOutputBuffer(media::VideoFrame::I420, dimensions);
335 uint8* yplane = NULL;
336 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
338 yplane = reinterpret_cast<uint8*>(buffer->data());
341 media::VideoFrame::PlaneAllocationSize(
342 media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions);
345 media::VideoFrame::PlaneAllocationSize(
346 media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions);
347 int yplane_stride = dimensions.width();
348 int uv_plane_stride = yplane_stride / 2;
351 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
353 libyuv::RotationMode rotation_mode = libyuv::kRotate0;
355 rotation_mode = libyuv::kRotate90;
356 else if (rotation == 180)
357 rotation_mode = libyuv::kRotate180;
358 else if (rotation == 270)
359 rotation_mode = libyuv::kRotate270;
361 switch (frame_format.pixel_format) {
362 case media::PIXEL_FORMAT_UNKNOWN: // Color format not set.
364 case media::PIXEL_FORMAT_I420:
365 DCHECK(!chopped_width && !chopped_height);
366 origin_colorspace = libyuv::FOURCC_I420;
368 case media::PIXEL_FORMAT_YV12:
369 DCHECK(!chopped_width && !chopped_height);
370 origin_colorspace = libyuv::FOURCC_YV12;
372 case media::PIXEL_FORMAT_NV21:
373 DCHECK(!chopped_width && !chopped_height);
374 origin_colorspace = libyuv::FOURCC_NV21;
376 case media::PIXEL_FORMAT_YUY2:
377 DCHECK(!chopped_width && !chopped_height);
378 origin_colorspace = libyuv::FOURCC_YUY2;
380 case media::PIXEL_FORMAT_UYVY:
381 DCHECK(!chopped_width && !chopped_height);
382 origin_colorspace = libyuv::FOURCC_UYVY;
384 case media::PIXEL_FORMAT_RGB24:
385 origin_colorspace = libyuv::FOURCC_24BG;
387 // TODO(wjia): Currently, for RGB24 on WIN, capture device always
388 // passes in positive src_width and src_height. Remove this hardcoded
389 // value when nagative src_height is supported. The negative src_height
390 // indicates that vertical flipping is needed.
394 case media::PIXEL_FORMAT_ARGB:
395 origin_colorspace = libyuv::FOURCC_ARGB;
397 case media::PIXEL_FORMAT_MJPEG:
398 origin_colorspace = libyuv::FOURCC_MJPG;
404 libyuv::ConvertToI420(data,
414 frame_format.frame_size.width(),
415 (flip ? -frame_format.frame_size.height() :
416 frame_format.frame_size.height()),
418 new_unrotated_height,
422 // Libyuv is not linked in for Android WebView builds, but video capture is
423 // not used in those builds either. Whenever libyuv is added in that build,
424 // address all these #ifdef parts, see http://crbug.com/299611 .
426 #endif // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
427 scoped_refptr<media::VideoFrame> frame =
428 media::VideoFrame::WrapExternalPackedMemory(
429 media::VideoFrame::I420,
431 gfx::Rect(dimensions),
434 media::VideoFrame::AllocationSize(media::VideoFrame::I420,
436 base::SharedMemory::NULLHandle(),
441 VideoCaptureFormat format(
442 dimensions, frame_format.frame_rate, media::PIXEL_FORMAT_I420);
443 BrowserThread::PostTask(
447 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
455 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
456 frame_format.frame_size.width());
457 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
458 frame_format.frame_size.height());
459 UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
460 frame_format.frame_size.width(),
461 frame_format.frame_size.height());
462 UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate",
463 frame_format.frame_rate);
464 UMA_HISTOGRAM_ENUMERATION("Media.VideoCapture.PixelFormat",
465 frame_format.pixel_format,
466 media::PIXEL_FORMAT_MAX);
467 first_frame_ = false;
472 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
473 const scoped_refptr<Buffer>& buffer,
474 const VideoCaptureFormat& buffer_format,
475 const scoped_refptr<media::VideoFrame>& frame,
476 base::TimeTicks timestamp) {
477 BrowserThread::PostTask(
481 &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
489 void VideoCaptureController::VideoCaptureDeviceClient::OnError(
490 const std::string& reason) {
491 MediaStreamManager::SendMessageToNativeLog(
492 "Error on video capture: " + reason);
493 BrowserThread::PostTask(BrowserThread::IO,
495 base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
498 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
499 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
500 media::VideoFrame::Format format,
501 const gfx::Size& dimensions) {
502 size_t frame_bytes = 0;
503 if (format == media::VideoFrame::NATIVE_TEXTURE) {
504 DCHECK_EQ(dimensions.width(), 0);
505 DCHECK_EQ(dimensions.height(), 0);
507 // The capture pipeline expects I420 for now.
508 DCHECK_EQ(format, media::VideoFrame::I420)
509 << "Non-I420 output buffer format " << format << " requested";
510 frame_bytes = media::VideoFrame::AllocationSize(format, dimensions);
513 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
515 buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
516 if (buffer_id == VideoCaptureBufferPool::kInvalidId)
520 buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
522 scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
523 new PoolBuffer(buffer_pool_, buffer_id, data, size));
525 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
526 BrowserThread::PostTask(BrowserThread::IO,
528 base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
529 controller_, buffer_id_to_drop));
532 return output_buffer;
535 VideoCaptureController::~VideoCaptureController() {
536 STLDeleteContainerPointers(controller_clients_.begin(),
537 controller_clients_.end());
540 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
541 const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
542 const media::VideoCaptureFormat& buffer_format,
543 const scoped_refptr<media::VideoFrame>& frame,
544 base::TimeTicks timestamp) {
545 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
546 DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
549 if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
550 for (ControllerClients::iterator client_it = controller_clients_.begin();
551 client_it != controller_clients_.end(); ++client_it) {
552 ControllerClient* client = *client_it;
553 if (client->session_closed)
556 if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
557 client->event_handler->OnMailboxBufferReady(client->controller_id,
559 *frame->mailbox_holder(),
563 bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
565 // On the first use of a buffer on a client, share the memory handle.
566 size_t memory_size = 0;
567 base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
568 buffer->id(), client->render_process_handle, &memory_size);
569 client->event_handler->OnBufferCreated(
570 client->controller_id, remote_handle, memory_size, buffer->id());
573 client->event_handler->OnBufferReady(
574 client->controller_id, buffer->id(), buffer_format, timestamp);
578 client->active_buffers.insert(std::make_pair(buffer->id(), frame))
580 DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
585 buffer_pool_->HoldForConsumers(buffer->id(), count);
588 void VideoCaptureController::DoErrorOnIOThread() {
589 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
590 state_ = VIDEO_CAPTURE_STATE_ERROR;
592 for (ControllerClients::iterator client_it = controller_clients_.begin();
593 client_it != controller_clients_.end(); ++client_it) {
594 ControllerClient* client = *client_it;
595 if (client->session_closed)
598 client->event_handler->OnError(client->controller_id);
602 void VideoCaptureController::DoBufferDestroyedOnIOThread(
603 int buffer_id_to_drop) {
604 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
606 for (ControllerClients::iterator client_it = controller_clients_.begin();
607 client_it != controller_clients_.end(); ++client_it) {
608 ControllerClient* client = *client_it;
609 if (client->session_closed)
612 if (client->known_buffers.erase(buffer_id_to_drop)) {
613 client->event_handler->OnBufferDestroyed(client->controller_id,
619 VideoCaptureController::ControllerClient*
620 VideoCaptureController::FindClient(
621 const VideoCaptureControllerID& id,
622 VideoCaptureControllerEventHandler* handler,
623 const ControllerClients& clients) {
624 for (ControllerClients::const_iterator client_it = clients.begin();
625 client_it != clients.end(); ++client_it) {
626 if ((*client_it)->controller_id == id &&
627 (*client_it)->event_handler == handler) {
634 VideoCaptureController::ControllerClient*
635 VideoCaptureController::FindClient(
637 const ControllerClients& clients) {
638 for (ControllerClients::const_iterator client_it = clients.begin();
639 client_it != clients.end(); ++client_it) {
640 if ((*client_it)->session_id == session_id) {
647 int VideoCaptureController::GetClientCount() {
648 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
649 return controller_clients_.size();
652 } // namespace content