content/browser/renderer_host/media/video_capture_controller.cc

   1 // Copyright 2012 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/renderer_host/media/video_capture_controller.h"
   6
   7 #include <stddef.h>
   8 #include <stdint.h>
   9
  10 #include <map>
  11 #include <set>
  12
  13 #include "base/command_line.h"
  14 #include "base/containers/contains.h"
  15 #include "base/functional/bind.h"
  16 #include "base/functional/callback_forward.h"
  17 #include "base/memory/raw_ptr.h"
  18 #include "base/metrics/histogram_functions.h"
  19 #include "base/metrics/histogram_macros.h"
  20 #include "base/ranges/algorithm.h"
  21 #include "base/token.h"
  22 #include "build/build_config.h"
  23 #include "content/browser/renderer_host/media/media_stream_manager.h"
  24 #include "content/browser/renderer_host/media/video_capture_manager.h"
  25 #include "content/public/browser/browser_thread.h"
  26 #include "content/public/common/content_switches.h"
  27 #include "media/base/video_frame.h"
  28 #include "media/base/video_types.h"
  29 #include "media/capture/mojom/video_capture_types.mojom.h"
  30 #include "media/capture/video/video_capture_buffer_pool.h"
  31 #include "media/capture/video/video_capture_buffer_tracker_factory_impl.h"
  32 #include "media/capture/video/video_capture_device_client.h"
  33 #include "media/capture/video/video_capture_metrics.h"
  34 #include "services/video_capture/public/mojom/video_effects_manager.mojom.h"
  35
  36 #if !BUILDFLAG(IS_ANDROID)
  37 #include "content/browser/compositor/image_transport_factory.h"
  38 #endif
  39
  40 using media::VideoCaptureFormat;
  41 using media::VideoFrame;
  42 using media::VideoFrameMetadata;
  43
  44 namespace content {
  45
  46 namespace {
  47
  48 // Counter used for identifying a DeviceRequest to start a capture device.
  49 static int g_device_start_id = 0;
  50
  51 static const int kInfiniteRatio = 99999;
  52
  53 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
  54   base::UmaHistogramSparse(                             \
  55       name, (height) ? ((width)*100) / (height) : kInfiniteRatio);
  56
  57 void CallOnError(media::VideoCaptureError error,
  58                  VideoCaptureControllerEventHandler* client,
  59                  const VideoCaptureControllerID& id) {
  60   client->OnError(id, error);
  61 }
  62
  63 void CallOnStarted(VideoCaptureControllerEventHandler* client,
  64                    const VideoCaptureControllerID& id) {
  65   client->OnStarted(id);
  66 }
  67
  68 void CallOnStartedUsingGpuDecode(VideoCaptureControllerEventHandler* client,
  69                                  const VideoCaptureControllerID& id) {
  70   client->OnStartedUsingGpuDecode(id);
  71 }
  72
  73 }  // anonymous namespace
  74
  75 struct VideoCaptureController::ControllerClient {
  76   ControllerClient(const VideoCaptureControllerID& id,
  77                    VideoCaptureControllerEventHandler* handler,
  78                    const media::VideoCaptureSessionId& session_id,
  79                    const media::VideoCaptureParams& params)
  80       : controller_id(id),
  81         event_handler(handler),
  82         session_id(session_id),
  83         parameters(params),
  84         session_closed(false),
  85         paused(false) {
  86 #if defined(TIZEN_MULTIMEDIA)
  87     paused = params.lazy_start;
  88 #endif
  89   }
  90
  91   ~ControllerClient() {}
  92
  93   // ID used for identifying this object.
  94   const VideoCaptureControllerID controller_id;
  95   const raw_ptr<VideoCaptureControllerEventHandler> event_handler;
  96
  97   const media::VideoCaptureSessionId session_id;
  98   const media::VideoCaptureParams parameters;
  99
 100   std::vector<int> known_buffer_context_ids;
 101   // |buffer_context_id|s of buffers currently being consumed by this client.
 102   std::vector<int> buffers_in_use;
 103
 104   // State of capture session, controlled by VideoCaptureManager directly. This
 105   // transitions to true as soon as StopSession() occurs, at which point the
 106   // client is sent an OnEnded() event. However, because the client retains a
 107   // VideoCaptureController* pointer, its ControllerClient entry lives on until
 108   // it unregisters itself via RemoveClient(), which may happen asynchronously.
 109   //
 110   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
 111   // OnEnded() events were processed synchronously (with the RemoveClient() done
 112   // implicitly), we could avoid tracking this state here in the Controller, and
 113   // simplify the code in both places.
 114   bool session_closed;
 115
 116   // Indicates whether the client is paused, if true, VideoCaptureController
 117   // stops updating its buffer.
 118   bool paused;
 119 };
 120
 121 VideoCaptureController::BufferContext::BufferContext(
 122     int buffer_context_id,
 123     int buffer_id,
 124     media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer,
 125     media::mojom::VideoBufferHandlePtr buffer_handle)
 126     : buffer_context_id_(buffer_context_id),
 127       buffer_id_(buffer_id),
 128       is_retired_(false),
 129       frame_feedback_id_(0),
 130       consumer_feedback_observer_(consumer_feedback_observer),
 131       buffer_handle_(std::move(buffer_handle)),
 132       consumer_hold_count_(0) {}
 133
 134 VideoCaptureController::BufferContext::~BufferContext() = default;
 135
 136 VideoCaptureController::BufferContext::BufferContext(
 137     VideoCaptureController::BufferContext&& other) = default;
 138
 139 VideoCaptureController::BufferContext& VideoCaptureController::BufferContext::
 140 operator=(BufferContext&& other) = default;
 141
 142 void VideoCaptureController::BufferContext::RecordConsumerUtilization(
 143     const media::VideoCaptureFeedback& feedback) {
 144   combined_consumer_feedback_.Combine(feedback);
 145 }
 146
 147 void VideoCaptureController::BufferContext::IncreaseConsumerCount() {
 148   consumer_hold_count_++;
 149 }
 150
 151 void VideoCaptureController::BufferContext::DecreaseConsumerCount() {
 152   consumer_hold_count_--;
 153   if (consumer_hold_count_ == 0) {
 154     if (consumer_feedback_observer_ != nullptr &&
 155         !combined_consumer_feedback_.Empty()) {
 156       // We set this now since frame_feedback_id_ may be updated at anytime.
 157       combined_consumer_feedback_.frame_id = frame_feedback_id_;
 158       consumer_feedback_observer_->OnUtilizationReport(
 159           combined_consumer_feedback_);
 160     }
 161     buffer_read_permission_.reset();
 162     combined_consumer_feedback_ = media::VideoCaptureFeedback();
 163   }
 164 }
 165
 166 media::mojom::VideoBufferHandlePtr
 167 VideoCaptureController::BufferContext::CloneBufferHandle() {
 168   if (buffer_handle_->is_unsafe_shmem_region()) {
 169     // Buffer handles are always writable as they come from
 170     // VideoCaptureBufferPool which, among other use cases, provides decoder
 171     // output buffers.
 172     //
 173     // TODO(crbug.com/793446): BroadcastingReceiver::BufferContext also defines
 174     // CloneBufferHandle and independently decides on handle permissions. The
 175     // permissions should be coordinated between these two classes.
 176     return media::mojom::VideoBufferHandle::NewUnsafeShmemRegion(
 177         buffer_handle_->get_unsafe_shmem_region().Duplicate());
 178   } else if (buffer_handle_->is_read_only_shmem_region()) {
 179     return media::mojom::VideoBufferHandle::NewReadOnlyShmemRegion(
 180         buffer_handle_->get_read_only_shmem_region().Duplicate());
 181   } else if (buffer_handle_->is_mailbox_handles()) {
 182     return media::mojom::VideoBufferHandle::NewMailboxHandles(
 183         buffer_handle_->get_mailbox_handles()->Clone());
 184   } else if (buffer_handle_->is_gpu_memory_buffer_handle()) {
 185     return media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
 186         buffer_handle_->get_gpu_memory_buffer_handle().Clone());
 187   } else {
 188     NOTREACHED() << "Unexpected video buffer handle type";
 189     return media::mojom::VideoBufferHandlePtr();
 190   }
 191 }
 192
 193 VideoCaptureController::VideoCaptureController(
 194     const std::string& device_id,
 195     blink::mojom::MediaStreamType stream_type,
 196     const media::VideoCaptureParams& params,
 197     std::unique_ptr<VideoCaptureDeviceLauncher> device_launcher,
 198     base::RepeatingCallback<void(const std::string&)> emit_log_message_cb)
 199     : serial_id_(g_device_start_id++),
 200       device_id_(device_id),
 201       stream_type_(stream_type),
 202       parameters_(params),
 203       device_launcher_(std::move(device_launcher)),
 204       emit_log_message_cb_(std::move(emit_log_message_cb)),
 205       device_launch_observer_(nullptr),
 206       state_(blink::VIDEO_CAPTURE_STATE_STARTING),
 207       has_received_frames_(false) {
 208   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 209 }
 210
 211 VideoCaptureController::~VideoCaptureController() = default;
 212
 213 base::WeakPtr<VideoCaptureController>
 214 VideoCaptureController::GetWeakPtrForIOThread() {
 215   return weak_ptr_factory_.GetWeakPtr();
 216 }
 217
 218 void VideoCaptureController::AddClient(
 219     const VideoCaptureControllerID& id,
 220     VideoCaptureControllerEventHandler* event_handler,
 221     const media::VideoCaptureSessionId& session_id,
 222     const media::VideoCaptureParams& params) {
 223   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 224   std::ostringstream string_stream;
 225   string_stream << "VideoCaptureController::AddClient(): id = " << id
 226                 << ", session_id = " << session_id.ToString()
 227                 << ", params.requested_format = "
 228                 << media::VideoCaptureFormat::ToString(params.requested_format);
 229   EmitLogMessage(string_stream.str(), 1);
 230
 231   // Params received from a renderer will have been validated by
 232   // VideoCaptureHost, so here we can just require validity.
 233   DCHECK(params.IsValid());
 234
 235   // Check that requested VideoCaptureParams are supported.  If not, report an
 236   // error immediately and punt.
 237   if (!(params.requested_format.pixel_format == media::PIXEL_FORMAT_I420 ||
 238         params.requested_format.pixel_format == media::PIXEL_FORMAT_Y16 ||
 239         params.requested_format.pixel_format == media::PIXEL_FORMAT_ARGB ||
 240         params.requested_format.pixel_format == media::PIXEL_FORMAT_NV12 ||
 241 #if BUILDFLAG(IS_TIZEN)
 242         params.requested_format.pixel_format == media::PIXEL_FORMAT_ENCODED ||
 243 #endif
 244 #if defined(TIZEN_MULTIMEDIA_MJPEG_SUPPORT)
 245         params.requested_format.pixel_format == media::PIXEL_FORMAT_MJPEG ||
 246 #endif
 247         params.requested_format.pixel_format == media::PIXEL_FORMAT_UNKNOWN)) {
 248     // Crash in debug builds since the renderer should not have asked for
 249     // unsupported parameters.
 250     LOG(DFATAL) << "Unsupported video capture parameters requested: "
 251                 << media::VideoCaptureFormat::ToString(params.requested_format);
 252     event_handler->OnError(id,
 253                            media::VideoCaptureError::
 254                                kVideoCaptureControllerUnsupportedPixelFormat);
 255     return;
 256   }
 257
 258   // If this is the first client added to the controller, cache the parameters.
 259   if (controller_clients_.empty())
 260     video_capture_format_ = params.requested_format;
 261
 262   // Signal error in case device is already in error state.
 263   if (state_ == blink::VIDEO_CAPTURE_STATE_ERROR) {
 264     event_handler->OnError(
 265         id,
 266         media::VideoCaptureError::kVideoCaptureControllerIsAlreadyInErrorState);
 267     return;
 268   }
 269
 270   // Do nothing if this client has called AddClient before.
 271   if (FindClient(id, event_handler, controller_clients_))
 272     return;
 273
 274   // If the device has reported OnStarted event, report it to this client here.
 275   if (state_ == blink::VIDEO_CAPTURE_STATE_STARTED)
 276     event_handler->OnStarted(id);
 277
 278   std::unique_ptr<ControllerClient> client =
 279       std::make_unique<ControllerClient>(id, event_handler, session_id, params);
 280   // If we already have gotten frame_info from the device, repeat it to the new
 281   // client.
 282   if (state_ != blink::VIDEO_CAPTURE_STATE_ERROR) {
 283     controller_clients_.push_back(std::move(client));
 284     base::UmaHistogramCounts100("Media.VideoCapture.NumberOfClients",
 285                                 controller_clients_.size());
 286   }
 287 }
 288
 289 base::UnguessableToken VideoCaptureController::RemoveClient(
 290     const VideoCaptureControllerID& id,
 291     VideoCaptureControllerEventHandler* event_handler) {
 292   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 293   std::ostringstream string_stream;
 294   string_stream << "VideoCaptureController::RemoveClient: id = " << id;
 295   EmitLogMessage(string_stream.str(), 1);
 296
 297   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 298   if (!client)
 299     return base::UnguessableToken();
 300
 301   for (const auto& buffer_id : client->buffers_in_use) {
 302     OnClientFinishedConsumingBuffer(client, buffer_id,
 303                                     media::VideoCaptureFeedback());
 304   }
 305   client->buffers_in_use.clear();
 306
 307   base::UnguessableToken session_id = client->session_id;
 308   controller_clients_.remove_if(
 309       [client](const std::unique_ptr<ControllerClient>& ptr) {
 310         return ptr.get() == client;
 311       });
 312
 313   return session_id;
 314 }
 315
 316 void VideoCaptureController::PauseClient(
 317     const VideoCaptureControllerID& id,
 318     VideoCaptureControllerEventHandler* event_handler) {
 319   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 320   DVLOG(1) << "VideoCaptureController::PauseClient: id = " << id;
 321
 322   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 323   if (!client)
 324     return;
 325
 326   DLOG_IF(WARNING, client->paused) << "Redundant client configuration";
 327
 328   client->paused = true;
 329 }
 330
 331 bool VideoCaptureController::ResumeClient(
 332     const VideoCaptureControllerID& id,
 333     VideoCaptureControllerEventHandler* event_handler) {
 334   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 335   DVLOG(1) << "VideoCaptureController::ResumeClient: id = " << id;
 336
 337   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 338   if (!client)
 339     return false;
 340
 341   if (!client->paused) {
 342     DVLOG(1) << "Calling resume on unpaused client";
 343     return false;
 344   }
 345
 346   client->paused = false;
 347   return true;
 348 }
 349
 350 size_t VideoCaptureController::GetClientCount() const {
 351   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 352   return controller_clients_.size();
 353 }
 354
 355 bool VideoCaptureController::HasActiveClient() const {
 356   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 357   for (const auto& client : controller_clients_) {
 358     if (!client->paused)
 359       return true;
 360   }
 361   return false;
 362 }
 363
 364 bool VideoCaptureController::HasPausedClient() const {
 365   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 366   for (const auto& client : controller_clients_) {
 367     if (client->paused)
 368       return true;
 369   }
 370   return false;
 371 }
 372
 373 void VideoCaptureController::StopSession(
 374     const base::UnguessableToken& session_id) {
 375   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 376   std::ostringstream string_stream;
 377   string_stream << "VideoCaptureController::StopSession: session_id = "
 378                 << session_id;
 379   EmitLogMessage(string_stream.str(), 1);
 380
 381   ControllerClient* client = FindClient(session_id, controller_clients_);
 382
 383   if (client) {
 384     client->session_closed = true;
 385     client->event_handler->OnEnded(client->controller_id);
 386   }
 387 }
 388
 389 void VideoCaptureController::ReturnBuffer(
 390     const VideoCaptureControllerID& id,
 391     VideoCaptureControllerEventHandler* event_handler,
 392     int buffer_id,
 393     const media::VideoCaptureFeedback& feedback) {
 394   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 395
 396   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 397   CHECK(client);
 398
 399   auto buffers_in_use_entry_iter =
 400       base::ranges::find(client->buffers_in_use, buffer_id);
 401   CHECK(buffers_in_use_entry_iter != std::end(client->buffers_in_use));
 402   client->buffers_in_use.erase(buffers_in_use_entry_iter);
 403
 404   OnClientFinishedConsumingBuffer(client, buffer_id, feedback);
 405 }
 406
 407 const absl::optional<media::VideoCaptureFormat>
 408 VideoCaptureController::GetVideoCaptureFormat() const {
 409   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 410   return video_capture_format_;
 411 }
 412
 413 void VideoCaptureController::OnCaptureConfigurationChanged() {
 414   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 415   EmitLogMessage(__func__, 3);
 416   for (const auto& client : controller_clients_) {
 417     if (client->session_closed) {
 418       continue;
 419     }
 420     client->event_handler->OnCaptureConfigurationChanged(client->controller_id);
 421   }
 422 }
 423
 424 void VideoCaptureController::OnNewBuffer(
 425     int32_t buffer_id,
 426     media::mojom::VideoBufferHandlePtr buffer_handle) {
 427   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 428   DCHECK(FindUnretiredBufferContextFromBufferId(buffer_id) ==
 429          buffer_contexts_.end());
 430   buffer_contexts_.emplace_back(next_buffer_context_id_++, buffer_id,
 431                                 launched_device_.get(),
 432                                 std::move(buffer_handle));
 433 }
 434
 435 void VideoCaptureController::OnFrameReadyInBuffer(
 436     media::ReadyFrameInBuffer frame) {
 437   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 438   DCHECK_NE(frame.buffer_id, media::VideoCaptureBufferPool::kInvalidId);
 439   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 440                "VideoCaptureController::OnFrameReadyInBuffer");
 441
 442   // Make ready buffers, get frame contexts and set their feedback IDs.
 443   // Transfer ownership of all the frame infos.
 444   BufferContext* frame_context;
 445   ReadyBuffer frame_ready_buffer = MakeReadyBufferAndSetContextFeedbackId(
 446       frame.buffer_id, frame.frame_feedback_id, std::move(frame.frame_info),
 447       &frame_context);
 448
 449   if (state_ != blink::VIDEO_CAPTURE_STATE_ERROR) {
 450     // Inform all active clients of the frames.
 451     for (const auto& client : controller_clients_) {
 452       if (client->session_closed || client->paused)
 453         continue;
 454       MakeClientUseBufferContext(frame_context, client.get());
 455       client->event_handler->OnBufferReady(client->controller_id,
 456                                            frame_ready_buffer);
 457     }
 458     // Transfer buffer read permissions to any contexts that now have consumers.
 459     if (frame_context->HasConsumers()) {
 460       frame_context->set_read_permission(
 461           std::move(frame.buffer_read_permission));
 462     }
 463   }
 464
 465   if (!has_received_frames_) {
 466     // Check the following group of metrics is captured only for cameras.
 467     if (stream_type() == blink::mojom::MediaStreamType::DEVICE_VIDEO_CAPTURE) {
 468       // This metric combines width and height into a single UMA metric.
 469       media::LogCaptureCurrentDeviceResolution(
 470           frame_ready_buffer.frame_info->coded_size.width(),
 471           frame_ready_buffer.frame_info->coded_size.height());
 472
 473       media::LogCaptureCurrentDevicePixelFormat(
 474           frame_ready_buffer.frame_info->pixel_format);
 475     }
 476     UMA_HISTOGRAM_COUNTS_1M("Media.VideoCapture.Width",
 477                             frame_ready_buffer.frame_info->coded_size.width());
 478     UMA_HISTOGRAM_COUNTS_1M("Media.VideoCapture.Height",
 479                             frame_ready_buffer.frame_info->coded_size.height());
 480     UMA_HISTOGRAM_ASPECT_RATIO(
 481         "Media.VideoCapture.AspectRatio",
 482         frame_ready_buffer.frame_info->coded_size.width(),
 483         frame_ready_buffer.frame_info->coded_size.height());
 484     double frame_rate = 0.0f;
 485     if (video_capture_format_) {
 486       frame_rate = frame_ready_buffer.frame_info->metadata.frame_rate.value_or(
 487           video_capture_format_->frame_rate);
 488     }
 489     UMA_HISTOGRAM_COUNTS_1M("Media.VideoCapture.FrameRate", frame_rate);
 490     UMA_HISTOGRAM_TIMES("Media.VideoCapture.DelayUntilFirstFrame",
 491                         base::TimeTicks::Now() - time_of_start_request_);
 492     OnLog("First frame received at VideoCaptureController");
 493     has_received_frames_ = true;
 494   }
 495 }
 496
 497 ReadyBuffer VideoCaptureController::MakeReadyBufferAndSetContextFeedbackId(
 498     int buffer_id,
 499     int frame_feedback_id,
 500     media::mojom::VideoFrameInfoPtr frame_info,
 501     BufferContext** out_buffer_context) {
 502   auto buffer_context_iter = FindUnretiredBufferContextFromBufferId(buffer_id);
 503   DCHECK(buffer_context_iter != buffer_contexts_.end());
 504   BufferContext* buffer_context = &(*buffer_context_iter);
 505   buffer_context->set_frame_feedback_id(frame_feedback_id);
 506   DCHECK(!buffer_context->HasConsumers());
 507   *out_buffer_context = buffer_context;
 508   return ReadyBuffer(buffer_context->buffer_context_id(),
 509                      std::move(frame_info));
 510 }
 511
 512 void VideoCaptureController::MakeClientUseBufferContext(
 513     BufferContext* frame_context,
 514     ControllerClient* client) {
 515   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 516   // On the first use of a BufferContext for a particular client, call
 517   // OnBufferCreated().
 518   if (!base::Contains(client->known_buffer_context_ids,
 519                       frame_context->buffer_context_id())) {
 520     client->known_buffer_context_ids.push_back(
 521         frame_context->buffer_context_id());
 522     client->event_handler->OnNewBuffer(client->controller_id,
 523                                        frame_context->CloneBufferHandle(),
 524                                        frame_context->buffer_context_id());
 525   }
 526   // Ensure buffer is registered as in use by the client.
 527   if (!base::Contains(client->buffers_in_use,
 528                       frame_context->buffer_context_id())) {
 529     client->buffers_in_use.push_back(frame_context->buffer_context_id());
 530   } else {
 531     NOTREACHED() << "Unexpected duplicate buffer: "
 532                  << frame_context->buffer_context_id();
 533   }
 534   frame_context->IncreaseConsumerCount();
 535 }
 536
 537 void VideoCaptureController::OnBufferRetired(int buffer_id) {
 538   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 539
 540   auto buffer_context_iter = FindUnretiredBufferContextFromBufferId(buffer_id);
 541   DCHECK(buffer_context_iter != buffer_contexts_.end());
 542
 543   // If there are any clients still using the buffer, we need to allow them
 544   // to finish up. We need to hold on to the BufferContext entry until then,
 545   // because it contains the consumer hold.
 546   if (!buffer_context_iter->HasConsumers())
 547     ReleaseBufferContext(buffer_context_iter);
 548   else
 549     buffer_context_iter->set_is_retired();
 550 }
 551
 552 void VideoCaptureController::OnError(media::VideoCaptureError error) {
 553   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 554   state_ = blink::VIDEO_CAPTURE_STATE_ERROR;
 555   PerformForClientsWithOpenSession(base::BindRepeating(&CallOnError, error));
 556 }
 557
 558 void VideoCaptureController::OnFrameDropped(
 559     media::VideoCaptureFrameDropReason reason) {
 560   // This method implements media::VideoFrameReceiver, which implements signals
 561   // between the capture process and browser process. We forward this call to
 562   // the renderer process where it eventually reached the MediaStreamVideoTrack.
 563   for (const auto& client : controller_clients_) {
 564     if (client->session_closed) {
 565       continue;
 566     }
 567     client->event_handler->OnFrameDropped(client->controller_id, reason);
 568   }
 569 }
 570
 571 void VideoCaptureController::OnNewSubCaptureTargetVersion(
 572     uint32_t sub_capture_target_version) {
 573   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 574   EmitLogMessage(
 575       base::StringPrintf("%s(%u)", __func__, sub_capture_target_version), 3);
 576   for (const auto& client : controller_clients_) {
 577     if (client->session_closed) {
 578       continue;
 579     }
 580     client->event_handler->OnNewSubCaptureTargetVersion(
 581         client->controller_id, sub_capture_target_version);
 582   }
 583 }
 584
 585 void VideoCaptureController::OnFrameWithEmptyRegionCapture() {
 586   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 587   EmitLogMessage(__func__, 3);
 588   for (const auto& client : controller_clients_) {
 589     if (client->session_closed) {
 590       continue;
 591     }
 592     client->event_handler->OnFrameWithEmptyRegionCapture(client->controller_id);
 593   }
 594 }
 595
 596 void VideoCaptureController::OnLog(const std::string& message) {
 597   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 598   EmitLogMessage(message, 3);
 599 }
 600
 601 void VideoCaptureController::OnStarted() {
 602   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 603   EmitLogMessage(__func__, 3);
 604   state_ = blink::VIDEO_CAPTURE_STATE_STARTED;
 605   PerformForClientsWithOpenSession(base::BindRepeating(&CallOnStarted));
 606 }
 607
 608 void VideoCaptureController::OnStartedUsingGpuDecode() {
 609   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 610   EmitLogMessage(__func__, 3);
 611   PerformForClientsWithOpenSession(
 612       base::BindRepeating(&CallOnStartedUsingGpuDecode));
 613 }
 614
 615 void VideoCaptureController::OnStopped() {
 616   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 617   EmitLogMessage(__func__, 3);
 618   // Clients of VideoCaptureController are currently not interested in
 619   // OnStopped events, so we simply swallow the event here. Note that, if we
 620   // wanted to forward it to clients in the future, care would have to be taken
 621   // for the case of there being outstanding OnBufferRetired() events that have
 622   // been deferred because a client was still consuming a retired buffer.
 623 }
 624
 625 void VideoCaptureController::OnDeviceLaunched(
 626     std::unique_ptr<LaunchedVideoCaptureDevice> device) {
 627   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 628   EmitLogMessage(__func__, 3);
 629   launched_device_ = std::move(device);
 630   for (auto& entry : buffer_contexts_)
 631     entry.set_consumer_feedback_observer(launched_device_.get());
 632   if (device_launch_observer_) {
 633     device_launch_observer_->OnDeviceLaunched(this);
 634   }
 635 }
 636
 637 void VideoCaptureController::OnDeviceLaunchFailed(
 638     media::VideoCaptureError error) {
 639   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 640   EmitLogMessage(__func__, 3);
 641   if (device_launch_observer_) {
 642     device_launch_observer_->OnDeviceLaunchFailed(this, error);
 643     device_launch_observer_ = nullptr;
 644   }
 645 }
 646
 647 void VideoCaptureController::OnDeviceLaunchAborted() {
 648   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 649   EmitLogMessage(__func__, 3);
 650   if (device_launch_observer_) {
 651     device_launch_observer_->OnDeviceLaunchAborted();
 652     device_launch_observer_ = nullptr;
 653   }
 654 }
 655
 656 void VideoCaptureController::OnDeviceConnectionLost() {
 657   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 658   EmitLogMessage(__func__, 3);
 659   if (device_launch_observer_) {
 660     device_launch_observer_->OnDeviceConnectionLost(this);
 661     device_launch_observer_ = nullptr;
 662   }
 663 }
 664
 665 void VideoCaptureController::CreateAndStartDeviceAsync(
 666     const media::VideoCaptureParams& params,
 667     VideoCaptureDeviceLaunchObserver* observer,
 668     base::OnceClosure done_cb,
 669     mojo::PendingRemote<video_capture::mojom::VideoEffectsManager>
 670         video_effects_manager) {
 671   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 672   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 673                "VideoCaptureController::CreateAndStartDeviceAsync");
 674   std::ostringstream string_stream;
 675   string_stream
 676       << "VideoCaptureController::CreateAndStartDeviceAsync: serial_id = "
 677       << serial_id() << ", device_id = " << device_id();
 678   EmitLogMessage(string_stream.str(), 1);
 679   time_of_start_request_ = base::TimeTicks::Now();
 680   device_launch_observer_ = observer;
 681   device_launcher_->LaunchDeviceAsync(
 682       device_id_, stream_type_, params, GetWeakPtrForIOThread(),
 683       base::BindOnce(&VideoCaptureController::OnDeviceConnectionLost,
 684                      GetWeakPtrForIOThread()),
 685       this, std::move(done_cb), std::move(video_effects_manager));
 686 }
 687
 688 void VideoCaptureController::ReleaseDeviceAsync(base::OnceClosure done_cb) {
 689   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 690   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 691                "VideoCaptureController::ReleaseDeviceAsync");
 692   std::ostringstream string_stream;
 693   string_stream << "VideoCaptureController::ReleaseDeviceAsync: serial_id = "
 694                 << serial_id() << ", device_id = " << device_id();
 695   EmitLogMessage(string_stream.str(), 1);
 696   if (!launched_device_) {
 697     device_launcher_->AbortLaunch();
 698     return;
 699   }
 700   // |buffer_contexts_| contain references to |launched_device_| as observers.
 701   // Clear those observer references prior to resetting |launced_device_|.
 702   for (auto& entry : buffer_contexts_)
 703     entry.set_consumer_feedback_observer(nullptr);
 704   launched_device_.reset();
 705 }
 706
 707 bool VideoCaptureController::IsDeviceAlive() const {
 708   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 709   return launched_device_ != nullptr;
 710 }
 711
 712 void VideoCaptureController::GetPhotoState(
 713     media::VideoCaptureDevice::GetPhotoStateCallback callback) const {
 714   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 715   DCHECK(launched_device_);
 716   launched_device_->GetPhotoState(std::move(callback));
 717 }
 718
 719 void VideoCaptureController::SetPhotoOptions(
 720     media::mojom::PhotoSettingsPtr settings,
 721     media::VideoCaptureDevice::SetPhotoOptionsCallback callback) {
 722   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 723   DCHECK(launched_device_);
 724   launched_device_->SetPhotoOptions(std::move(settings), std::move(callback));
 725 }
 726
 727 void VideoCaptureController::TakePhoto(
 728     media::VideoCaptureDevice::TakePhotoCallback callback) {
 729   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 730   DCHECK(launched_device_);
 731   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 732                        "VideoCaptureController::TakePhoto",
 733                        TRACE_EVENT_SCOPE_PROCESS);
 734   launched_device_->TakePhoto(std::move(callback));
 735 }
 736
 737 void VideoCaptureController::MaybeSuspend() {
 738   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 739   DCHECK(launched_device_);
 740   EmitLogMessage(__func__, 3);
 741   launched_device_->MaybeSuspendDevice();
 742 }
 743
 744 void VideoCaptureController::Resume() {
 745   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 746   DCHECK(launched_device_);
 747   EmitLogMessage(__func__, 3);
 748   launched_device_->ResumeDevice();
 749 }
 750
 751 void VideoCaptureController::Crop(
 752     const base::Token& crop_id,
 753     uint32_t sub_capture_target_version,
 754     base::OnceCallback<void(media::mojom::ApplySubCaptureTargetResult)>
 755         callback) {
 756   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 757   DCHECK(launched_device_);
 758
 759   EmitLogMessage(__func__, 3);
 760
 761   was_crop_ever_called_ = true;
 762
 763   if (controller_clients_.size() != 1) {
 764     std::move(callback).Run(
 765         media::mojom::ApplySubCaptureTargetResult::kNotImplemented);
 766     return;
 767   }
 768
 769   launched_device_->Crop(crop_id, sub_capture_target_version,
 770                          std::move(callback));
 771 }
 772
 773 void VideoCaptureController::RequestRefreshFrame() {
 774   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 775   DCHECK(launched_device_);
 776   launched_device_->RequestRefreshFrame();
 777 }
 778
 779 void VideoCaptureController::SetDesktopCaptureWindowIdAsync(
 780     gfx::NativeViewId window_id,
 781     base::OnceClosure done_cb) {
 782   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 783   DCHECK(launched_device_);
 784   launched_device_->SetDesktopCaptureWindowIdAsync(window_id,
 785                                                    std::move(done_cb));
 786 }
 787
 788 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 789     const VideoCaptureControllerID& id,
 790     VideoCaptureControllerEventHandler* handler,
 791     const ControllerClients& clients) {
 792   for (const auto& client : clients) {
 793     if (client->controller_id == id && client->event_handler == handler)
 794       return client.get();
 795   }
 796   return nullptr;
 797 }
 798
 799 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
 800     const base::UnguessableToken& session_id,
 801     const ControllerClients& clients) {
 802   for (const auto& client : clients) {
 803     if (client->session_id == session_id)
 804       return client.get();
 805   }
 806   return nullptr;
 807 }
 808
 809 std::vector<VideoCaptureController::BufferContext>::iterator
 810 VideoCaptureController::FindBufferContextFromBufferContextId(
 811     int buffer_context_id) {
 812   return base::ranges::find(buffer_contexts_, buffer_context_id,
 813                             &BufferContext::buffer_context_id);
 814 }
 815
 816 std::vector<VideoCaptureController::BufferContext>::iterator
 817 VideoCaptureController::FindUnretiredBufferContextFromBufferId(int buffer_id) {
 818   return base::ranges::find_if(
 819       buffer_contexts_, [buffer_id](const BufferContext& entry) {
 820         return (entry.buffer_id() == buffer_id) && !entry.is_retired();
 821       });
 822 }
 823
 824 void VideoCaptureController::OnClientFinishedConsumingBuffer(
 825     ControllerClient* client,
 826     int buffer_context_id,
 827     const media::VideoCaptureFeedback& feedback) {
 828   auto buffer_context_iter =
 829       FindBufferContextFromBufferContextId(buffer_context_id);
 830   DCHECK(buffer_context_iter != buffer_contexts_.end());
 831
 832   buffer_context_iter->RecordConsumerUtilization(feedback);
 833   buffer_context_iter->DecreaseConsumerCount();
 834   if (!buffer_context_iter->HasConsumers() &&
 835       buffer_context_iter->is_retired()) {
 836     ReleaseBufferContext(buffer_context_iter);
 837   }
 838 }
 839
 840 void VideoCaptureController::ReleaseBufferContext(
 841     const std::vector<BufferContext>::iterator& buffer_context_iter) {
 842   for (const auto& client : controller_clients_) {
 843     if (client->session_closed)
 844       continue;
 845     auto entry_iter =
 846         base::ranges::find(client->known_buffer_context_ids,
 847                            buffer_context_iter->buffer_context_id());
 848     if (entry_iter != std::end(client->known_buffer_context_ids)) {
 849       client->known_buffer_context_ids.erase(entry_iter);
 850       client->event_handler->OnBufferDestroyed(
 851           client->controller_id, buffer_context_iter->buffer_context_id());
 852     }
 853   }
 854   buffer_contexts_.erase(buffer_context_iter);
 855 }
 856
 857 void VideoCaptureController::PerformForClientsWithOpenSession(
 858     EventHandlerAction action) {
 859   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 860   for (const auto& client : controller_clients_) {
 861     if (client->session_closed)
 862       continue;
 863     action.Run(client->event_handler.get(), client->controller_id);
 864   }
 865 }
 866
 867 void VideoCaptureController::EmitLogMessage(const std::string& message,
 868                                             int verbose_log_level) {
 869   DVLOG(verbose_log_level) << message;
 870   emit_log_message_cb_.Run(message);
 871 }
 872
 873 }  // namespace content