media/capture/video/video_capture_device_client.cc

   1 // Copyright 2015 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/capture/video/video_capture_device_client.h"
   6
   7 #include <memory>
   8 #include <utility>
   9
  10 #include "base/command_line.h"
  11 #include "base/containers/contains.h"
  12 #include "base/feature_list.h"
  13 #include "base/functional/bind.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/ranges/algorithm.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "build/build_config.h"
  20 #include "build/chromeos_buildflags.h"
  21 #include "media/base/video_frame.h"
  22 #include "media/capture/video/scoped_buffer_pool_reservation.h"
  23 #include "media/capture/video/video_capture_buffer_handle.h"
  24 #include "media/capture/video/video_capture_buffer_pool.h"
  25 #include "media/capture/video/video_frame_receiver.h"
  26 #include "media/capture/video_capture_types.h"
  27 #include "third_party/libyuv/include/libyuv.h"
  28
  29 #if BUILDFLAG(IS_CHROMEOS_ASH)
  30 #include "media/capture/video/chromeos/video_capture_jpeg_decoder.h"
  31 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
  32
  33 namespace {
  34
  35 bool IsFormatSupported(media::VideoPixelFormat pixel_format) {
  36   return (pixel_format == media::PIXEL_FORMAT_I420 ||
  37           // NV12 and MJPEG are used by GpuMemoryBuffer on Chrome OS.
  38           pixel_format == media::PIXEL_FORMAT_NV12 ||
  39           pixel_format == media::PIXEL_FORMAT_MJPEG ||
  40           pixel_format == media::PIXEL_FORMAT_Y16);
  41 }
  42
  43 libyuv::RotationMode TranslateRotation(int rotation_degrees) {
  44   CHECK_EQ(0, rotation_degrees % 90)
  45       << " Rotation must be a multiple of 90, now: " << rotation_degrees;
  46   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
  47   if (rotation_degrees == 90)
  48     rotation_mode = libyuv::kRotate90;
  49   else if (rotation_degrees == 180)
  50     rotation_mode = libyuv::kRotate180;
  51   else if (rotation_degrees == 270)
  52     rotation_mode = libyuv::kRotate270;
  53   return rotation_mode;
  54 }
  55
  56 void GetI420BufferAccess(
  57     const media::VideoCaptureDevice::Client::Buffer& buffer,
  58     const gfx::Size& dimensions,
  59     uint8_t** y_plane_data,
  60     uint8_t** u_plane_data,
  61     uint8_t** v_plane_data,
  62     int* y_plane_stride,
  63     int* uv_plane_stride) {
  64   *y_plane_data = buffer.handle_provider->GetHandleForInProcessAccess()->data();
  65   *u_plane_data = *y_plane_data + media::VideoFrame::PlaneSize(
  66                                       media::PIXEL_FORMAT_I420,
  67                                       media::VideoFrame::kYPlane, dimensions)
  68                                       .GetArea();
  69   *v_plane_data = *u_plane_data + media::VideoFrame::PlaneSize(
  70                                       media::PIXEL_FORMAT_I420,
  71                                       media::VideoFrame::kUPlane, dimensions)
  72                                       .GetArea();
  73   *y_plane_stride = dimensions.width();
  74   *uv_plane_stride = *y_plane_stride / 2;
  75 }
  76
  77 gfx::ColorSpace OverrideColorSpaceForLibYuvConversion(
  78     const gfx::ColorSpace& color_space,
  79     const media::VideoPixelFormat pixel_format) {
  80   gfx::ColorSpace overriden_color_space = color_space;
  81   switch (pixel_format) {
  82     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
  83       break;
  84     case media::PIXEL_FORMAT_ARGB:
  85     case media::PIXEL_FORMAT_XRGB:
  86     case media::PIXEL_FORMAT_RGB24:
  87     case media::PIXEL_FORMAT_ABGR:
  88     case media::PIXEL_FORMAT_XBGR:
  89       // Check if we can merge data 's primary and transfer function into the
  90       // returned color space.
  91       if (color_space.IsValid()) {
  92         // The raw data is rgb so we expect its color space to only hold gamma
  93         // correction.
  94         DCHECK(color_space == color_space.GetAsFullRangeRGB());
  95
  96         // This captured ARGB data is going to be converted to yuv using libyuv
  97         // ConvertToI420 which internally uses Rec601 coefficients. So build a
  98         // combined colorspace that contains both the above gamma correction
  99         // and the yuv conversion information.
 100         // TODO(julien.isorce): instead pass color space information to libyuv
 101         // once the support is added, see http://crbug.com/libyuv/835.
 102         overriden_color_space = color_space.GetWithMatrixAndRange(
 103             gfx::ColorSpace::MatrixID::SMPTE170M,
 104             gfx::ColorSpace::RangeID::LIMITED);
 105       } else {
 106         // Color space is not specified but it is probably safe to assume it is
 107         // sRGB though, and so it would be valid to assume that libyuv's
 108         // ConvertToI420() is going to produce results in Rec601, or very close
 109         // to it.
 110         overriden_color_space = gfx::ColorSpace::CreateREC601();
 111       }
 112       break;
 113     default:
 114       break;
 115   }
 116
 117   return overriden_color_space;
 118 }
 119
 120 struct FourccAndFlip {
 121   libyuv::FourCC fourcc_format = libyuv::FOURCC_ANY;
 122   bool flip = false;
 123 };
 124
 125 FourccAndFlip GetFourccAndFlipFromPixelFormat(
 126     const media::VideoCaptureFormat& format,
 127     bool flip_y) {
 128   const int is_width_odd = format.frame_size.width() & 1;
 129   const int is_height_odd = format.frame_size.height() & 1;
 130
 131   switch (format.pixel_format) {
 132     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
 133       return {};
 134     case media::PIXEL_FORMAT_I420:
 135       CHECK(!is_width_odd && !is_height_odd);
 136       return {libyuv::FOURCC_I420};
 137     case media::PIXEL_FORMAT_YV12:
 138       CHECK(!is_width_odd && !is_height_odd);
 139       return {libyuv::FOURCC_YV12};
 140     case media::PIXEL_FORMAT_NV12:
 141       CHECK(!is_width_odd && !is_height_odd);
 142       return {libyuv::FOURCC_NV12};
 143     case media::PIXEL_FORMAT_NV21:
 144       CHECK(!is_width_odd && !is_height_odd);
 145       return {libyuv::FOURCC_NV21};
 146     case media::PIXEL_FORMAT_YUY2:
 147       CHECK(!is_width_odd && !is_height_odd);
 148       return {libyuv::FOURCC_YUY2};
 149     case media::PIXEL_FORMAT_UYVY:
 150       CHECK(!is_width_odd && !is_height_odd);
 151       return {libyuv::FOURCC_UYVY};
 152     case media::PIXEL_FORMAT_RGB24:
 153       if constexpr (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)) {
 154         // Linux RGB24 defines red at lowest byte address,
 155         // see http://linuxtv.org/downloads/v4l-dvb-apis/packed-rgb.html.
 156         return {libyuv::FOURCC_RAW};
 157       } else if constexpr (BUILDFLAG(IS_WIN)) {
 158         // Windows RGB24 defines blue at lowest byte,
 159         // see https://msdn.microsoft.com/en-us/library/windows/desktop/dd407253
 160
 161         // TODO(wjia): Currently, for RGB24 on WIN, capture device always passes
 162         // in positive src_width and src_height. Remove this hardcoded value
 163         // when negative src_height is supported. The negative src_height
 164         // indicates that vertical flipping is needed.
 165         return {libyuv::FOURCC_24BG, true};
 166       } else {
 167         NOTREACHED_NORETURN()
 168             << "RGB24 is only available in Linux and Windows platforms";
 169       }
 170     case media::PIXEL_FORMAT_ARGB:
 171       // Windows platforms e.g. send the data vertically flipped sometimes.
 172       return {libyuv::FOURCC_ARGB, flip_y};
 173     case media::PIXEL_FORMAT_MJPEG:
 174       return {libyuv::FOURCC_MJPG};
 175     default:
 176       NOTREACHED_NORETURN();
 177   }
 178 }
 179
 180 }  // anonymous namespace
 181
 182 namespace media {
 183
 184 #if BUILDFLAG(IS_MAC)
 185 // TODO(https://crbug.com/1474871): When this code path has been verified on
 186 // Canary, change to enabled-by-default.
 187 BASE_FEATURE(kFallbackToSharedMemoryIfNotNv12OnMac,
 188              "FallbackToSharedMemoryIfNotNv12OnMac",
 189              base::FEATURE_DISABLED_BY_DEFAULT);
 190 #endif
 191
 192 namespace {
 193
 194 class ScopedAccessPermissionEndWithCallback
 195     : public VideoCaptureDevice::Client::Buffer::ScopedAccessPermission {
 196  public:
 197   explicit ScopedAccessPermissionEndWithCallback(base::OnceClosure closure)
 198       : closure_(std::move(closure)) {}
 199   ~ScopedAccessPermissionEndWithCallback() override {
 200     std::move(closure_).Run();
 201   }
 202
 203  private:
 204   base::OnceClosure closure_;
 205 };
 206
 207 }  // anonymous namespace
 208
 209 class BufferPoolBufferHandleProvider
 210     : public VideoCaptureDevice::Client::Buffer::HandleProvider {
 211  public:
 212   BufferPoolBufferHandleProvider(
 213       scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 214       int buffer_id)
 215       : buffer_pool_(std::move(buffer_pool)), buffer_id_(buffer_id) {}
 216
 217   base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() override {
 218     return buffer_pool_->DuplicateAsUnsafeRegion(buffer_id_);
 219   }
 220   gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() override {
 221     return buffer_pool_->GetGpuMemoryBufferHandle(buffer_id_);
 222   }
 223   std::unique_ptr<VideoCaptureBufferHandle> GetHandleForInProcessAccess()
 224       override {
 225     return buffer_pool_->GetHandleForInProcessAccess(buffer_id_);
 226   }
 227
 228  private:
 229   const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
 230   const int buffer_id_;
 231 };
 232
 233 #if BUILDFLAG(IS_CHROMEOS_ASH)
 234 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
 235     VideoCaptureBufferType target_buffer_type,
 236     std::unique_ptr<VideoFrameReceiver> receiver,
 237     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 238     VideoCaptureJpegDecoderFactoryCB optional_jpeg_decoder_factory_callback)
 239     : target_buffer_type_(target_buffer_type),
 240       receiver_(std::move(receiver)),
 241       optional_jpeg_decoder_factory_callback_(
 242           std::move(optional_jpeg_decoder_factory_callback)),
 243       buffer_pool_(std::move(buffer_pool)),
 244       last_captured_pixel_format_(PIXEL_FORMAT_UNKNOWN) {
 245   on_started_using_gpu_cb_ =
 246       base::BindOnce(&VideoFrameReceiver::OnStartedUsingGpuDecode,
 247                      base::Unretained(receiver_.get()));
 248 }
 249 #else
 250 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
 251     VideoCaptureBufferType target_buffer_type,
 252     std::unique_ptr<VideoFrameReceiver> receiver,
 253     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 254     mojo::PendingRemote<video_capture::mojom::VideoEffectsManager>
 255         video_effects_manager)
 256     : target_buffer_type_(target_buffer_type),
 257       receiver_(std::move(receiver)),
 258       buffer_pool_(std::move(buffer_pool)),
 259       last_captured_pixel_format_(PIXEL_FORMAT_UNKNOWN),
 260       mojo_task_runner_(base::SequencedTaskRunner::GetCurrentDefault()),
 261       effects_manager_(std::move(video_effects_manager), mojo_task_runner_) {}
 262 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 263
 264 VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
 265 #if !BUILDFLAG(IS_CHROMEOS_ASH)
 266   // Make sure that the remote is destroyed from the same sequence that it was
 267   // created on.
 268   mojo_task_runner_->PostTask(
 269       FROM_HERE, base::DoNothingWithBoundArgs(std::move(effects_manager_)));
 270 #endif  // !BUILDFLAG(IS_CHROMEOS_ASH)
 271   for (int buffer_id : buffer_ids_known_by_receiver_) {
 272     receiver_->OnBufferRetired(buffer_id);
 273   }
 274   receiver_->OnStopped();
 275 }
 276
 277 // static
 278 VideoCaptureDevice::Client::Buffer VideoCaptureDeviceClient::MakeBufferStruct(
 279     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 280     int buffer_id,
 281     int frame_feedback_id) {
 282   return Buffer(
 283       buffer_id, frame_feedback_id,
 284       std::make_unique<BufferPoolBufferHandleProvider>(buffer_pool, buffer_id),
 285       std::make_unique<ScopedBufferPoolReservation<ProducerReleaseTraits>>(
 286           buffer_pool, buffer_id));
 287 }
 288
 289 void VideoCaptureDeviceClient::OnCaptureConfigurationChanged() {
 290   receiver_->OnCaptureConfigurationChanged();
 291 }
 292
 293 void VideoCaptureDeviceClient::OnIncomingCapturedData(
 294     const uint8_t* data,
 295     int length,
 296     const VideoCaptureFormat& format,
 297     const gfx::ColorSpace& data_color_space,
 298     int rotation,
 299     bool flip_y,
 300     base::TimeTicks reference_time,
 301     base::TimeDelta timestamp,
 302     int frame_feedback_id) {
 303   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 304   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 305                "VideoCaptureDeviceClient::OnIncomingCapturedData");
 306
 307   // The input |length| can be greater than the required buffer size because of
 308   // paddings and/or alignments, but it cannot be smaller.
 309   CHECK_GE(static_cast<size_t>(length),
 310            media::VideoFrame::AllocationSize(format.pixel_format,
 311                                              format.frame_size));
 312
 313   if (last_captured_pixel_format_ != format.pixel_format) {
 314     OnLog("Pixel format: " + VideoPixelFormatToString(format.pixel_format));
 315     last_captured_pixel_format_ = format.pixel_format;
 316
 317 #if BUILDFLAG(IS_CHROMEOS_ASH)
 318     if (format.pixel_format == PIXEL_FORMAT_MJPEG &&
 319         optional_jpeg_decoder_factory_callback_) {
 320       external_jpeg_decoder_ =
 321           std::move(optional_jpeg_decoder_factory_callback_).Run();
 322       CHECK(external_jpeg_decoder_);
 323       external_jpeg_decoder_->Initialize();
 324     }
 325 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 326   }
 327
 328   if (!format.IsValid()) {
 329     receiver_->OnFrameDropped(
 330         VideoCaptureFrameDropReason::kDeviceClientFrameHasInvalidFormat);
 331     return;
 332   }
 333
 334   if (format.pixel_format == PIXEL_FORMAT_Y16) {
 335     return OnIncomingCapturedY16Data(data, length, format, reference_time,
 336                                      timestamp, frame_feedback_id);
 337   }
 338
 339   // |new_unrotated_{width,height}| are the dimensions of the output buffer that
 340   // satisfy the video pixel format requirements. For I420, this is equivalent
 341   // to rounding to nearest even number (away from zero, eg 13 becomes 14, -13
 342   // becomes -14).
 343   const int new_unrotated_width = format.frame_size.width() & ~1;
 344   const int new_unrotated_height = format.frame_size.height() & ~1;
 345
 346   int destination_width = new_unrotated_width;
 347   int destination_height = new_unrotated_height;
 348   if (rotation == 90 || rotation == 270)
 349     std::swap(destination_width, destination_height);
 350
 351   const gfx::Size dimensions(destination_width, destination_height);
 352   CHECK(dimensions.height());
 353   CHECK(dimensions.width());
 354
 355   const libyuv::RotationMode rotation_mode = TranslateRotation(rotation);
 356
 357   Buffer buffer;
 358   auto reservation_result_code = ReserveOutputBuffer(
 359       dimensions, PIXEL_FORMAT_I420, frame_feedback_id, &buffer);
 360   if (reservation_result_code != ReserveResult::kSucceeded) {
 361     receiver_->OnFrameDropped(
 362         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 363     return;
 364   }
 365
 366   const auto [fourcc_format, flip] =
 367       GetFourccAndFlipFromPixelFormat(format, flip_y);
 368
 369   uint8_t* y_plane_data;
 370   uint8_t* u_plane_data;
 371   uint8_t* v_plane_data;
 372   int yplane_stride, uv_plane_stride;
 373   GetI420BufferAccess(buffer, dimensions, &y_plane_data, &u_plane_data,
 374                       &v_plane_data, &yplane_stride, &uv_plane_stride);
 375
 376   const gfx::ColorSpace color_space = OverrideColorSpaceForLibYuvConversion(
 377       data_color_space, format.pixel_format);
 378
 379 #if BUILDFLAG(IS_CHROMEOS_ASH)
 380   if (external_jpeg_decoder_) {
 381     const VideoCaptureJpegDecoder::STATUS status =
 382         external_jpeg_decoder_->GetStatus();
 383     if (status == VideoCaptureJpegDecoder::FAILED) {
 384       external_jpeg_decoder_.reset();
 385     } else if (status == VideoCaptureJpegDecoder::INIT_PASSED &&
 386                format.pixel_format == PIXEL_FORMAT_MJPEG && rotation == 0 &&
 387                !flip) {
 388       if (on_started_using_gpu_cb_)
 389         std::move(on_started_using_gpu_cb_).Run();
 390       external_jpeg_decoder_->DecodeCapturedData(
 391           data, length, format, reference_time, timestamp, std::move(buffer));
 392       return;
 393     }
 394   }
 395 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 396
 397   // libyuv::ConvertToI420 uses Rec601 to convert RGB to YUV.
 398   if (libyuv::ConvertToI420(
 399           data, length, y_plane_data, yplane_stride, u_plane_data,
 400           uv_plane_stride, v_plane_data, uv_plane_stride, /*crop_x=*/0,
 401           /*crop_y=*/0, format.frame_size.width(),
 402           (flip ? -1 : 1) * format.frame_size.height(), new_unrotated_width,
 403           new_unrotated_height, rotation_mode, fourcc_format) != 0) {
 404     DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
 405                   << VideoPixelFormatToString(format.pixel_format);
 406     receiver_->OnFrameDropped(
 407         VideoCaptureFrameDropReason::kDeviceClientLibyuvConvertToI420Failed);
 408     return;
 409   }
 410
 411   const VideoCaptureFormat output_format =
 412       VideoCaptureFormat(dimensions, format.frame_rate, PIXEL_FORMAT_I420);
 413   OnIncomingCapturedBufferExt(std::move(buffer), output_format, color_space,
 414                               reference_time, timestamp, gfx::Rect(dimensions),
 415                               VideoFrameMetadata());
 416 }
 417
 418 void VideoCaptureDeviceClient::OnIncomingCapturedGfxBuffer(
 419     gfx::GpuMemoryBuffer* buffer,
 420     const VideoCaptureFormat& frame_format,
 421     int clockwise_rotation,
 422     base::TimeTicks reference_time,
 423     base::TimeDelta timestamp,
 424     int frame_feedback_id) {
 425   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 426                "VideoCaptureDeviceClient::OnIncomingCapturedGfxBuffer");
 427
 428   if (last_captured_pixel_format_ != frame_format.pixel_format) {
 429     OnLog("Pixel format: " +
 430           VideoPixelFormatToString(frame_format.pixel_format));
 431     last_captured_pixel_format_ = frame_format.pixel_format;
 432   }
 433
 434   if (!frame_format.IsValid()) {
 435     receiver_->OnFrameDropped(
 436         VideoCaptureFrameDropReason::kDeviceClientFrameHasInvalidFormat);
 437     return;
 438   }
 439
 440   int destination_width = buffer->GetSize().width();
 441   int destination_height = buffer->GetSize().height();
 442   if (clockwise_rotation == 90 || clockwise_rotation == 270)
 443     std::swap(destination_width, destination_height);
 444
 445   libyuv::RotationMode rotation_mode = TranslateRotation(clockwise_rotation);
 446
 447   const gfx::Size dimensions(destination_width, destination_height);
 448   Buffer output_buffer;
 449   const auto reservation_result_code = ReserveOutputBuffer(
 450       dimensions, PIXEL_FORMAT_I420, frame_feedback_id, &output_buffer);
 451
 452   // Failed to reserve I420 output buffer, so drop the frame.
 453   if (reservation_result_code != ReserveResult::kSucceeded) {
 454     receiver_->OnFrameDropped(
 455         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 456     return;
 457   }
 458
 459   uint8_t* y_plane_data;
 460   uint8_t* u_plane_data;
 461   uint8_t* v_plane_data;
 462   int y_plane_stride, uv_plane_stride;
 463   GetI420BufferAccess(output_buffer, dimensions, &y_plane_data, &u_plane_data,
 464                       &v_plane_data, &y_plane_stride, &uv_plane_stride);
 465
 466   if (!buffer->Map()) {
 467     LOG(ERROR) << "Failed to map GPU memory buffer";
 468     receiver_->OnFrameDropped(
 469         VideoCaptureFrameDropReason::kGpuMemoryBufferMapFailed);
 470     return;
 471   }
 472   base::ScopedClosureRunner unmap_closure(
 473       base::BindOnce([](gfx::GpuMemoryBuffer* buffer) { buffer->Unmap(); },
 474                      base::Unretained(buffer)));
 475
 476   int ret = -EINVAL;
 477   switch (frame_format.pixel_format) {
 478     case PIXEL_FORMAT_NV12:
 479       ret = libyuv::NV12ToI420Rotate(
 480           reinterpret_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
 481           reinterpret_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
 482           y_plane_data, y_plane_stride, u_plane_data, uv_plane_stride,
 483           v_plane_data, uv_plane_stride, buffer->GetSize().width(),
 484           buffer->GetSize().height(), rotation_mode);
 485       break;
 486
 487     default:
 488       LOG(ERROR) << "Unsupported format: "
 489                  << VideoPixelFormatToString(frame_format.pixel_format);
 490   }
 491   if (ret) {
 492     DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
 493                   << VideoPixelFormatToString(frame_format.pixel_format);
 494     receiver_->OnFrameDropped(
 495         VideoCaptureFrameDropReason::kDeviceClientLibyuvConvertToI420Failed);
 496     return;
 497   }
 498
 499   const VideoCaptureFormat output_format = VideoCaptureFormat(
 500       dimensions, frame_format.frame_rate, PIXEL_FORMAT_I420);
 501   OnIncomingCapturedBuffer(std::move(output_buffer), output_format,
 502                            reference_time, timestamp);
 503 }
 504
 505 void VideoCaptureDeviceClient::OnIncomingCapturedExternalBuffer(
 506     CapturedExternalVideoBuffer buffer,
 507     base::TimeTicks reference_time,
 508     base::TimeDelta timestamp,
 509     const gfx::Rect& visible_rect) {
 510   ReadyFrameInBuffer ready_frame;
 511   if (CreateReadyFrameFromExternalBuffer(
 512           std::move(buffer), reference_time, timestamp, visible_rect,
 513           &ready_frame) != ReserveResult::kSucceeded) {
 514     DVLOG(2) << __func__
 515              << " CreateReadyFrameFromExternalBuffer failed: reservation "
 516                 "trakcer failed.";
 517     return;
 518   }
 519   receiver_->OnFrameReadyInBuffer(std::move(ready_frame));
 520 }
 521
 522 VideoCaptureDevice::Client::ReserveResult
 523 VideoCaptureDeviceClient::CreateReadyFrameFromExternalBuffer(
 524     CapturedExternalVideoBuffer buffer,
 525     base::TimeTicks reference_time,
 526     base::TimeDelta timestamp,
 527     const gfx::Rect& visible_rect,
 528     ReadyFrameInBuffer* ready_buffer) {
 529   // Reserve an ID for this buffer that will not conflict with any of the IDs
 530   // used by |buffer_pool_|.
 531   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
 532   int buffer_id = VideoCaptureBufferPool::kInvalidId;
 533   // Use std::move to transfer the handle ownership here since the buffer will
 534   // be created and confirm each ScopedHandle can only have one owner at a
 535   // time. Because the subsequent code mojom::VideoFrameInfoPtr needs to use the
 536   // buffer information, so here use |buffer_for_reserve_id| instead of
 537   // std::move(buffer).
 538   CapturedExternalVideoBuffer buffer_for_reserve_id =
 539       CapturedExternalVideoBuffer(std::move(buffer.handle), buffer.format,
 540                                   buffer.color_space);
 541 #if BUILDFLAG(IS_WIN)
 542   buffer_for_reserve_id.imf_buffer = std::move(buffer.imf_buffer);
 543 #endif
 544   VideoCaptureDevice::Client::ReserveResult reservation_result_code =
 545       buffer_pool_->ReserveIdForExternalBuffer(std::move(buffer_for_reserve_id),
 546                                                visible_rect.size(),
 547                                                &buffer_id_to_drop, &buffer_id);
 548   // If a buffer to retire was specified, retire one.
 549   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
 550     auto entry_iter =
 551         base::ranges::find(buffer_ids_known_by_receiver_, buffer_id_to_drop);
 552     if (entry_iter != buffer_ids_known_by_receiver_.end()) {
 553       buffer_ids_known_by_receiver_.erase(entry_iter);
 554       receiver_->OnBufferRetired(buffer_id_to_drop);
 555     }
 556   }
 557
 558   if (reservation_result_code != ReserveResult::kSucceeded) {
 559     return reservation_result_code;
 560   }
 561
 562   // Register the buffer with the receiver if it is new.
 563   if (!base::Contains(buffer_ids_known_by_receiver_, buffer_id)) {
 564     // On windows, 'GetGpuMemoryBufferHandle' will duplicate a new handle which
 565     // refers to the same object as the original handle.
 566     // https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-duplicatehandle
 567     media::mojom::VideoBufferHandlePtr buffer_handle =
 568         media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
 569             buffer_pool_->GetGpuMemoryBufferHandle(buffer_id));
 570     receiver_->OnNewBuffer(buffer_id, std::move(buffer_handle));
 571     buffer_ids_known_by_receiver_.push_back(buffer_id);
 572   }
 573
 574   // Construct the ready frame, to be passed on to the |receiver_| by the caller
 575   // of this method.
 576   mojom::VideoFrameInfoPtr info = mojom::VideoFrameInfo::New();
 577   info->timestamp = timestamp;
 578   info->pixel_format = buffer.format.pixel_format;
 579   info->color_space = buffer.color_space;
 580   info->coded_size = buffer.format.frame_size;
 581   info->visible_rect = visible_rect;
 582   info->metadata.frame_rate = buffer.format.frame_rate;
 583   info->metadata.reference_time = reference_time;
 584
 585   buffer_pool_->HoldForConsumers(buffer_id, 1);
 586   buffer_pool_->RelinquishProducerReservation(buffer_id);
 587
 588   *ready_buffer = ReadyFrameInBuffer(
 589       buffer_id, 0 /* frame_feedback_id */,
 590       std::make_unique<ScopedBufferPoolReservation<ConsumerReleaseTraits>>(
 591           buffer_pool_, buffer_id),
 592       std::move(info));
 593   return VideoCaptureDevice::Client::ReserveResult::kSucceeded;
 594 }
 595
 596 VideoCaptureDevice::Client::ReserveResult
 597 VideoCaptureDeviceClient::ReserveOutputBuffer(const gfx::Size& frame_size,
 598                                               VideoPixelFormat pixel_format,
 599                                               int frame_feedback_id,
 600                                               Buffer* buffer) {
 601   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 602   CHECK_GT(frame_size.width(), 0);
 603   CHECK_GT(frame_size.height(), 0);
 604   CHECK(IsFormatSupported(pixel_format));
 605
 606   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
 607   int buffer_id = VideoCaptureBufferPool::kInvalidId;
 608   auto reservation_result_code = buffer_pool_->ReserveForProducer(
 609       frame_size, pixel_format, nullptr, frame_feedback_id, &buffer_id,
 610       &buffer_id_to_drop);
 611   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
 612     // |buffer_pool_| has decided to release a buffer. Notify receiver in case
 613     // the buffer has already been shared with it.
 614     auto entry_iter =
 615         base::ranges::find(buffer_ids_known_by_receiver_, buffer_id_to_drop);
 616     if (entry_iter != buffer_ids_known_by_receiver_.end()) {
 617       buffer_ids_known_by_receiver_.erase(entry_iter);
 618       receiver_->OnBufferRetired(buffer_id_to_drop);
 619     }
 620   }
 621   if (reservation_result_code != ReserveResult::kSucceeded) {
 622     DVLOG(2) << __func__ << " reservation failed";
 623     return reservation_result_code;
 624   }
 625
 626   CHECK_NE(VideoCaptureBufferPool::kInvalidId, buffer_id);
 627
 628   if (!base::Contains(buffer_ids_known_by_receiver_, buffer_id)) {
 629     VideoCaptureBufferType target_buffer_type = target_buffer_type_;
 630 #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC)
 631     // If MediaFoundationD3D11VideoCapture or VideoCaptureDeviceAVFoundation
 632     // fails to produce NV12 as is expected on these platforms when the target
 633     // buffer type is `kGpuMemoryBuffer`, a shared memory buffer may be sent
 634     // instead.
 635     if (target_buffer_type == VideoCaptureBufferType::kGpuMemoryBuffer &&
 636         pixel_format != PIXEL_FORMAT_NV12
 637 #if BUILDFLAG(IS_MAC)
 638         && base::FeatureList::IsEnabled(kFallbackToSharedMemoryIfNotNv12OnMac)
 639 #endif
 640     ) {
 641       target_buffer_type = VideoCaptureBufferType::kSharedMemory;
 642     }
 643 #endif
 644     media::mojom::VideoBufferHandlePtr buffer_handle;
 645     switch (target_buffer_type) {
 646       case VideoCaptureBufferType::kSharedMemory:
 647         buffer_handle = media::mojom::VideoBufferHandle::NewUnsafeShmemRegion(
 648             buffer_pool_->DuplicateAsUnsafeRegion(buffer_id));
 649         break;
 650       case VideoCaptureBufferType::kMailboxHolder:
 651         NOTREACHED();
 652         break;
 653       case VideoCaptureBufferType::kGpuMemoryBuffer:
 654         buffer_handle =
 655             media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
 656                 buffer_pool_->GetGpuMemoryBufferHandle(buffer_id));
 657         break;
 658     }
 659     receiver_->OnNewBuffer(buffer_id, std::move(buffer_handle));
 660     buffer_ids_known_by_receiver_.push_back(buffer_id);
 661   }
 662
 663   *buffer = MakeBufferStruct(buffer_pool_, buffer_id, frame_feedback_id);
 664   return ReserveResult::kSucceeded;
 665 }
 666
 667 void VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
 668     Buffer buffer,
 669     const VideoCaptureFormat& format,
 670     base::TimeTicks reference_time,
 671     base::TimeDelta timestamp) {
 672   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 673   OnIncomingCapturedBufferExt(
 674       std::move(buffer), format, gfx::ColorSpace(), reference_time, timestamp,
 675       gfx::Rect(format.frame_size), VideoFrameMetadata());
 676 }
 677
 678 void VideoCaptureDeviceClient::OnIncomingCapturedBufferExt(
 679     Buffer buffer,
 680     const VideoCaptureFormat& format,
 681     const gfx::ColorSpace& color_space,
 682     base::TimeTicks reference_time,
 683     base::TimeDelta timestamp,
 684     gfx::Rect visible_rect,
 685     const VideoFrameMetadata& additional_metadata) {
 686   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 687                "VideoCaptureDeviceClient::OnIncomingCapturedBufferExt");
 688   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 689
 690   VideoFrameMetadata metadata = additional_metadata;
 691   metadata.frame_rate = format.frame_rate;
 692   metadata.reference_time = reference_time;
 693
 694   mojom::VideoFrameInfoPtr info = mojom::VideoFrameInfo::New();
 695   info->timestamp = timestamp;
 696   info->pixel_format = format.pixel_format;
 697   info->color_space = color_space;
 698   info->coded_size = format.frame_size;
 699   info->visible_rect = visible_rect;
 700   info->metadata = metadata;
 701   info->is_premapped = buffer.is_premapped;
 702
 703   buffer_pool_->HoldForConsumers(buffer.id, 1);
 704   receiver_->OnFrameReadyInBuffer(ReadyFrameInBuffer(
 705       buffer.id, buffer.frame_feedback_id,
 706       std::make_unique<ScopedBufferPoolReservation<ConsumerReleaseTraits>>(
 707           buffer_pool_, buffer.id),
 708       std::move(info)));
 709 }
 710
 711 void VideoCaptureDeviceClient::OnError(VideoCaptureError error,
 712                                        const base::Location& from_here,
 713                                        const std::string& reason) {
 714   const std::string log_message = base::StringPrintf(
 715       "error@ %s, %s, OS message: %s", from_here.ToString().c_str(),
 716       reason.c_str(),
 717       logging::SystemErrorCodeToString(logging::GetLastSystemErrorCode())
 718           .c_str());
 719   DLOG(ERROR) << log_message;
 720   OnLog(log_message);
 721   receiver_->OnError(error);
 722 }
 723
 724 void VideoCaptureDeviceClient::OnFrameDropped(
 725     VideoCaptureFrameDropReason reason) {
 726   receiver_->OnFrameDropped(reason);
 727 }
 728
 729 void VideoCaptureDeviceClient::OnLog(const std::string& message) {
 730   receiver_->OnLog(message);
 731 }
 732
 733 void VideoCaptureDeviceClient::OnStarted() {
 734   receiver_->OnStarted();
 735 }
 736
 737 double VideoCaptureDeviceClient::GetBufferPoolUtilization() const {
 738   return buffer_pool_->GetBufferPoolUtilization();
 739 }
 740
 741 void VideoCaptureDeviceClient::OnIncomingCapturedY16Data(
 742     const uint8_t* data,
 743     int length,
 744     const VideoCaptureFormat& format,
 745     base::TimeTicks reference_time,
 746     base::TimeDelta timestamp,
 747     int frame_feedback_id) {
 748   Buffer buffer;
 749   const auto reservation_result_code = ReserveOutputBuffer(
 750       format.frame_size, PIXEL_FORMAT_Y16, frame_feedback_id, &buffer);
 751   // The input |length| can be greater than the required buffer size because of
 752   // paddings and/or alignments, but it cannot be smaller.
 753   CHECK_GE(static_cast<size_t>(length),
 754            media::VideoFrame::AllocationSize(format.pixel_format,
 755                                              format.frame_size));
 756   // Failed to reserve output buffer, so drop the frame.
 757   if (reservation_result_code != ReserveResult::kSucceeded) {
 758     receiver_->OnFrameDropped(
 759         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 760     return;
 761   }
 762   auto buffer_access = buffer.handle_provider->GetHandleForInProcessAccess();
 763   memcpy(buffer_access->data(), data, length);
 764   const VideoCaptureFormat output_format = VideoCaptureFormat(
 765       format.frame_size, format.frame_rate, PIXEL_FORMAT_Y16);
 766   OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time,
 767                            timestamp);
 768 }
 769 }  // namespace media