media/capture/video/video_capture_device_client.cc

   1 // Copyright 2015 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/capture/video/video_capture_device_client.h"
   6
   7 #include <memory>
   8 #include <utility>
   9
  10 #include "base/command_line.h"
  11 #include "base/containers/contains.h"
  12 #include "base/feature_list.h"
  13 #include "base/functional/bind.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/location.h"
  16 #include "base/ranges/algorithm.h"
  17 #include "base/strings/stringprintf.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "build/build_config.h"
  20 #include "build/chromeos_buildflags.h"
  21 #include "media/base/video_frame.h"
  22 #include "media/capture/video/scoped_buffer_pool_reservation.h"
  23 #include "media/capture/video/video_capture_buffer_handle.h"
  24 #include "media/capture/video/video_capture_buffer_pool.h"
  25 #include "media/capture/video/video_frame_receiver.h"
  26 #include "media/capture/video_capture_types.h"
  27 #include "third_party/libyuv/include/libyuv.h"
  28
  29 #if BUILDFLAG(IS_CHROMEOS_ASH)
  30 #include "media/capture/video/chromeos/video_capture_jpeg_decoder.h"
  31 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
  32
  33 namespace {
  34
  35 bool IsFormatSupported(media::VideoPixelFormat pixel_format) {
  36   return (pixel_format == media::PIXEL_FORMAT_I420 ||
  37           // NV12 and MJPEG are used by GpuMemoryBuffer on Chrome OS.
  38           pixel_format == media::PIXEL_FORMAT_NV12 ||
  39           pixel_format == media::PIXEL_FORMAT_MJPEG ||
  40 #if BUILDFLAG(IS_TIZEN)
  41           pixel_format == media::PIXEL_FORMAT_ENCODED ||
  42 #endif
  43           pixel_format == media::PIXEL_FORMAT_Y16);
  44 }
  45
  46 libyuv::RotationMode TranslateRotation(int rotation_degrees) {
  47   CHECK_EQ(0, rotation_degrees % 90)
  48       << " Rotation must be a multiple of 90, now: " << rotation_degrees;
  49   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
  50   if (rotation_degrees == 90)
  51     rotation_mode = libyuv::kRotate90;
  52   else if (rotation_degrees == 180)
  53     rotation_mode = libyuv::kRotate180;
  54   else if (rotation_degrees == 270)
  55     rotation_mode = libyuv::kRotate270;
  56   return rotation_mode;
  57 }
  58
  59 void GetI420BufferAccess(
  60     const media::VideoCaptureDevice::Client::Buffer& buffer,
  61     const gfx::Size& dimensions,
  62     uint8_t** y_plane_data,
  63     uint8_t** u_plane_data,
  64     uint8_t** v_plane_data,
  65     int* y_plane_stride,
  66     int* uv_plane_stride) {
  67   *y_plane_data = buffer.handle_provider->GetHandleForInProcessAccess()->data();
  68   *u_plane_data = *y_plane_data + media::VideoFrame::PlaneSize(
  69                                       media::PIXEL_FORMAT_I420,
  70                                       media::VideoFrame::kYPlane, dimensions)
  71                                       .GetArea();
  72   *v_plane_data = *u_plane_data + media::VideoFrame::PlaneSize(
  73                                       media::PIXEL_FORMAT_I420,
  74                                       media::VideoFrame::kUPlane, dimensions)
  75                                       .GetArea();
  76   *y_plane_stride = dimensions.width();
  77   *uv_plane_stride = *y_plane_stride / 2;
  78 }
  79
  80 gfx::ColorSpace OverrideColorSpaceForLibYuvConversion(
  81     const gfx::ColorSpace& color_space,
  82     const media::VideoPixelFormat pixel_format) {
  83   gfx::ColorSpace overriden_color_space = color_space;
  84   switch (pixel_format) {
  85     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
  86       break;
  87     case media::PIXEL_FORMAT_ARGB:
  88     case media::PIXEL_FORMAT_XRGB:
  89     case media::PIXEL_FORMAT_RGB24:
  90     case media::PIXEL_FORMAT_ABGR:
  91     case media::PIXEL_FORMAT_XBGR:
  92       // Check if we can merge data 's primary and transfer function into the
  93       // returned color space.
  94       if (color_space.IsValid()) {
  95         // The raw data is rgb so we expect its color space to only hold gamma
  96         // correction.
  97         DCHECK(color_space == color_space.GetAsFullRangeRGB());
  98
  99         // This captured ARGB data is going to be converted to yuv using libyuv
 100         // ConvertToI420 which internally uses Rec601 coefficients. So build a
 101         // combined colorspace that contains both the above gamma correction
 102         // and the yuv conversion information.
 103         // TODO(julien.isorce): instead pass color space information to libyuv
 104         // once the support is added, see http://crbug.com/libyuv/835.
 105         overriden_color_space = color_space.GetWithMatrixAndRange(
 106             gfx::ColorSpace::MatrixID::SMPTE170M,
 107             gfx::ColorSpace::RangeID::LIMITED);
 108       } else {
 109         // Color space is not specified but it is probably safe to assume it is
 110         // sRGB though, and so it would be valid to assume that libyuv's
 111         // ConvertToI420() is going to produce results in Rec601, or very close
 112         // to it.
 113         overriden_color_space = gfx::ColorSpace::CreateREC601();
 114       }
 115       break;
 116     default:
 117       break;
 118   }
 119
 120   return overriden_color_space;
 121 }
 122
 123 struct FourccAndFlip {
 124   libyuv::FourCC fourcc_format = libyuv::FOURCC_ANY;
 125   bool flip = false;
 126 };
 127
 128 FourccAndFlip GetFourccAndFlipFromPixelFormat(
 129     const media::VideoCaptureFormat& format,
 130     bool flip_y) {
 131   const int is_width_odd = format.frame_size.width() & 1;
 132   const int is_height_odd = format.frame_size.height() & 1;
 133
 134   switch (format.pixel_format) {
 135     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
 136       return {};
 137     case media::PIXEL_FORMAT_I420:
 138       CHECK(!is_width_odd && !is_height_odd);
 139       return {libyuv::FOURCC_I420};
 140     case media::PIXEL_FORMAT_YV12:
 141       CHECK(!is_width_odd && !is_height_odd);
 142       return {libyuv::FOURCC_YV12};
 143     case media::PIXEL_FORMAT_NV12:
 144       CHECK(!is_width_odd && !is_height_odd);
 145       return {libyuv::FOURCC_NV12};
 146     case media::PIXEL_FORMAT_NV21:
 147       CHECK(!is_width_odd && !is_height_odd);
 148       return {libyuv::FOURCC_NV21};
 149     case media::PIXEL_FORMAT_YUY2:
 150       CHECK(!is_width_odd && !is_height_odd);
 151       return {libyuv::FOURCC_YUY2};
 152     case media::PIXEL_FORMAT_UYVY:
 153       CHECK(!is_width_odd && !is_height_odd);
 154       return {libyuv::FOURCC_UYVY};
 155     case media::PIXEL_FORMAT_RGB24:
 156       if constexpr (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)) {
 157         // Linux RGB24 defines red at lowest byte address,
 158         // see http://linuxtv.org/downloads/v4l-dvb-apis/packed-rgb.html.
 159         return {libyuv::FOURCC_RAW};
 160       } else if constexpr (BUILDFLAG(IS_WIN)) {
 161         // Windows RGB24 defines blue at lowest byte,
 162         // see https://msdn.microsoft.com/en-us/library/windows/desktop/dd407253
 163
 164         // TODO(wjia): Currently, for RGB24 on WIN, capture device always passes
 165         // in positive src_width and src_height. Remove this hardcoded value
 166         // when negative src_height is supported. The negative src_height
 167         // indicates that vertical flipping is needed.
 168         return {libyuv::FOURCC_24BG, true};
 169       } else {
 170         NOTREACHED_NORETURN()
 171             << "RGB24 is only available in Linux and Windows platforms";
 172       }
 173     case media::PIXEL_FORMAT_ARGB:
 174       // Windows platforms e.g. send the data vertically flipped sometimes.
 175       return {libyuv::FOURCC_ARGB, flip_y};
 176     case media::PIXEL_FORMAT_MJPEG:
 177       return {libyuv::FOURCC_MJPG};
 178     default:
 179       NOTREACHED_NORETURN();
 180   }
 181 }
 182
 183 }  // anonymous namespace
 184
 185 namespace media {
 186
 187 #if BUILDFLAG(IS_MAC)
 188 // TODO(https://crbug.com/1474871): When this code path has been verified on
 189 // Canary, change to enabled-by-default.
 190 BASE_FEATURE(kFallbackToSharedMemoryIfNotNv12OnMac,
 191              "FallbackToSharedMemoryIfNotNv12OnMac",
 192              base::FEATURE_DISABLED_BY_DEFAULT);
 193 #endif
 194
 195 namespace {
 196
 197 class ScopedAccessPermissionEndWithCallback
 198     : public VideoCaptureDevice::Client::Buffer::ScopedAccessPermission {
 199  public:
 200   explicit ScopedAccessPermissionEndWithCallback(base::OnceClosure closure)
 201       : closure_(std::move(closure)) {}
 202   ~ScopedAccessPermissionEndWithCallback() override {
 203     std::move(closure_).Run();
 204   }
 205
 206  private:
 207   base::OnceClosure closure_;
 208 };
 209
 210 }  // anonymous namespace
 211
 212 class BufferPoolBufferHandleProvider
 213     : public VideoCaptureDevice::Client::Buffer::HandleProvider {
 214  public:
 215   BufferPoolBufferHandleProvider(
 216       scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 217       int buffer_id)
 218       : buffer_pool_(std::move(buffer_pool)), buffer_id_(buffer_id) {}
 219
 220   base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() override {
 221     return buffer_pool_->DuplicateAsUnsafeRegion(buffer_id_);
 222   }
 223   gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() override {
 224     return buffer_pool_->GetGpuMemoryBufferHandle(buffer_id_);
 225   }
 226   std::unique_ptr<VideoCaptureBufferHandle> GetHandleForInProcessAccess()
 227       override {
 228     return buffer_pool_->GetHandleForInProcessAccess(buffer_id_);
 229   }
 230
 231  private:
 232   const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
 233   const int buffer_id_;
 234 };
 235
 236 #if BUILDFLAG(IS_CHROMEOS_ASH)
 237 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
 238     VideoCaptureBufferType target_buffer_type,
 239     std::unique_ptr<VideoFrameReceiver> receiver,
 240     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 241     VideoCaptureJpegDecoderFactoryCB optional_jpeg_decoder_factory_callback)
 242     : target_buffer_type_(target_buffer_type),
 243       receiver_(std::move(receiver)),
 244       optional_jpeg_decoder_factory_callback_(
 245           std::move(optional_jpeg_decoder_factory_callback)),
 246       buffer_pool_(std::move(buffer_pool)),
 247       last_captured_pixel_format_(PIXEL_FORMAT_UNKNOWN) {
 248   on_started_using_gpu_cb_ =
 249       base::BindOnce(&VideoFrameReceiver::OnStartedUsingGpuDecode,
 250                      base::Unretained(receiver_.get()));
 251 }
 252 #else
 253 VideoCaptureDeviceClient::VideoCaptureDeviceClient(
 254     VideoCaptureBufferType target_buffer_type,
 255     std::unique_ptr<VideoFrameReceiver> receiver,
 256     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 257     mojo::PendingRemote<video_capture::mojom::VideoEffectsManager>
 258         video_effects_manager)
 259     : target_buffer_type_(target_buffer_type),
 260       receiver_(std::move(receiver)),
 261       buffer_pool_(std::move(buffer_pool)),
 262       last_captured_pixel_format_(PIXEL_FORMAT_UNKNOWN),
 263       mojo_task_runner_(base::SequencedTaskRunner::GetCurrentDefault()),
 264       effects_manager_(std::move(video_effects_manager), mojo_task_runner_) {}
 265 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 266
 267 VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {
 268 #if !BUILDFLAG(IS_CHROMEOS_ASH)
 269   // Make sure that the remote is destroyed from the same sequence that it was
 270   // created on.
 271   mojo_task_runner_->PostTask(
 272       FROM_HERE, base::DoNothingWithBoundArgs(std::move(effects_manager_)));
 273 #endif  // !BUILDFLAG(IS_CHROMEOS_ASH)
 274   for (int buffer_id : buffer_ids_known_by_receiver_) {
 275     receiver_->OnBufferRetired(buffer_id);
 276   }
 277   receiver_->OnStopped();
 278 }
 279
 280 // static
 281 VideoCaptureDevice::Client::Buffer VideoCaptureDeviceClient::MakeBufferStruct(
 282     scoped_refptr<VideoCaptureBufferPool> buffer_pool,
 283     int buffer_id,
 284     int frame_feedback_id) {
 285   return Buffer(
 286       buffer_id, frame_feedback_id,
 287       std::make_unique<BufferPoolBufferHandleProvider>(buffer_pool, buffer_id),
 288       std::make_unique<ScopedBufferPoolReservation<ProducerReleaseTraits>>(
 289           buffer_pool, buffer_id));
 290 }
 291
 292 void VideoCaptureDeviceClient::OnCaptureConfigurationChanged() {
 293   receiver_->OnCaptureConfigurationChanged();
 294 }
 295
 296 void VideoCaptureDeviceClient::OnIncomingCapturedData(
 297     const uint8_t* data,
 298     int length,
 299     const VideoCaptureFormat& format,
 300     const gfx::ColorSpace& data_color_space,
 301     int rotation,
 302     bool flip_y,
 303     base::TimeTicks reference_time,
 304     base::TimeDelta timestamp,
 305     int frame_feedback_id) {
 306   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 307   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 308                "VideoCaptureDeviceClient::OnIncomingCapturedData");
 309
 310   // The input |length| can be greater than the required buffer size because of
 311   // paddings and/or alignments, but it cannot be smaller.
 312   CHECK_GE(static_cast<size_t>(length),
 313            media::VideoFrame::AllocationSize(format.pixel_format,
 314                                              format.frame_size));
 315
 316   if (last_captured_pixel_format_ != format.pixel_format) {
 317     OnLog("Pixel format: " + VideoPixelFormatToString(format.pixel_format));
 318     last_captured_pixel_format_ = format.pixel_format;
 319
 320 #if BUILDFLAG(IS_CHROMEOS_ASH)
 321     if (format.pixel_format == PIXEL_FORMAT_MJPEG &&
 322         optional_jpeg_decoder_factory_callback_) {
 323       external_jpeg_decoder_ =
 324           std::move(optional_jpeg_decoder_factory_callback_).Run();
 325       CHECK(external_jpeg_decoder_);
 326       external_jpeg_decoder_->Initialize();
 327     }
 328 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 329   }
 330
 331   if (!format.IsValid()) {
 332     receiver_->OnFrameDropped(
 333         VideoCaptureFrameDropReason::kDeviceClientFrameHasInvalidFormat);
 334     return;
 335   }
 336
 337   if (format.pixel_format == PIXEL_FORMAT_Y16) {
 338     return OnIncomingCapturedY16Data(data, length, format, reference_time,
 339                                      timestamp, frame_feedback_id);
 340   }
 341
 342   // |new_unrotated_{width,height}| are the dimensions of the output buffer that
 343   // satisfy the video pixel format requirements. For I420, this is equivalent
 344   // to rounding to nearest even number (away from zero, eg 13 becomes 14, -13
 345   // becomes -14).
 346   const int new_unrotated_width = format.frame_size.width() & ~1;
 347   const int new_unrotated_height = format.frame_size.height() & ~1;
 348
 349   int destination_width = new_unrotated_width;
 350   int destination_height = new_unrotated_height;
 351   if (rotation == 90 || rotation == 270)
 352     std::swap(destination_width, destination_height);
 353
 354   const gfx::Size dimensions(destination_width, destination_height);
 355   CHECK(dimensions.height());
 356   CHECK(dimensions.width());
 357
 358   const libyuv::RotationMode rotation_mode = TranslateRotation(rotation);
 359
 360   Buffer buffer;
 361   auto reservation_result_code = ReserveOutputBuffer(
 362       dimensions, PIXEL_FORMAT_I420, frame_feedback_id, &buffer);
 363   if (reservation_result_code != ReserveResult::kSucceeded) {
 364     receiver_->OnFrameDropped(
 365         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 366     return;
 367   }
 368
 369   const auto [fourcc_format, flip] =
 370       GetFourccAndFlipFromPixelFormat(format, flip_y);
 371
 372   uint8_t* y_plane_data;
 373   uint8_t* u_plane_data;
 374   uint8_t* v_plane_data;
 375   int yplane_stride, uv_plane_stride;
 376   GetI420BufferAccess(buffer, dimensions, &y_plane_data, &u_plane_data,
 377                       &v_plane_data, &yplane_stride, &uv_plane_stride);
 378
 379   const gfx::ColorSpace color_space = OverrideColorSpaceForLibYuvConversion(
 380       data_color_space, format.pixel_format);
 381
 382 #if BUILDFLAG(IS_CHROMEOS_ASH)
 383   if (external_jpeg_decoder_) {
 384     const VideoCaptureJpegDecoder::STATUS status =
 385         external_jpeg_decoder_->GetStatus();
 386     if (status == VideoCaptureJpegDecoder::FAILED) {
 387       external_jpeg_decoder_.reset();
 388     } else if (status == VideoCaptureJpegDecoder::INIT_PASSED &&
 389                format.pixel_format == PIXEL_FORMAT_MJPEG && rotation == 0 &&
 390                !flip) {
 391       if (on_started_using_gpu_cb_)
 392         std::move(on_started_using_gpu_cb_).Run();
 393       external_jpeg_decoder_->DecodeCapturedData(
 394           data, length, format, reference_time, timestamp, std::move(buffer));
 395       return;
 396     }
 397   }
 398 #endif  // BUILDFLAG(IS_CHROMEOS_ASH)
 399
 400   // libyuv::ConvertToI420 uses Rec601 to convert RGB to YUV.
 401   if (libyuv::ConvertToI420(
 402           data, length, y_plane_data, yplane_stride, u_plane_data,
 403           uv_plane_stride, v_plane_data, uv_plane_stride, /*crop_x=*/0,
 404           /*crop_y=*/0, format.frame_size.width(),
 405           (flip ? -1 : 1) * format.frame_size.height(), new_unrotated_width,
 406           new_unrotated_height, rotation_mode, fourcc_format) != 0) {
 407     DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
 408                   << VideoPixelFormatToString(format.pixel_format);
 409     receiver_->OnFrameDropped(
 410         VideoCaptureFrameDropReason::kDeviceClientLibyuvConvertToI420Failed);
 411     return;
 412   }
 413
 414   const VideoCaptureFormat output_format =
 415       VideoCaptureFormat(dimensions, format.frame_rate, PIXEL_FORMAT_I420);
 416   OnIncomingCapturedBufferExt(std::move(buffer), output_format, color_space,
 417                               reference_time, timestamp, gfx::Rect(dimensions),
 418                               VideoFrameMetadata());
 419 }
 420
 421 void VideoCaptureDeviceClient::OnIncomingCapturedGfxBuffer(
 422     gfx::GpuMemoryBuffer* buffer,
 423     const VideoCaptureFormat& frame_format,
 424     int clockwise_rotation,
 425     base::TimeTicks reference_time,
 426     base::TimeDelta timestamp,
 427     int frame_feedback_id) {
 428   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 429                "VideoCaptureDeviceClient::OnIncomingCapturedGfxBuffer");
 430
 431   if (last_captured_pixel_format_ != frame_format.pixel_format) {
 432     OnLog("Pixel format: " +
 433           VideoPixelFormatToString(frame_format.pixel_format));
 434     last_captured_pixel_format_ = frame_format.pixel_format;
 435   }
 436
 437   if (!frame_format.IsValid()) {
 438     receiver_->OnFrameDropped(
 439         VideoCaptureFrameDropReason::kDeviceClientFrameHasInvalidFormat);
 440     return;
 441   }
 442
 443   int destination_width = buffer->GetSize().width();
 444   int destination_height = buffer->GetSize().height();
 445   if (clockwise_rotation == 90 || clockwise_rotation == 270)
 446     std::swap(destination_width, destination_height);
 447
 448   libyuv::RotationMode rotation_mode = TranslateRotation(clockwise_rotation);
 449
 450   const gfx::Size dimensions(destination_width, destination_height);
 451   Buffer output_buffer;
 452   const auto reservation_result_code = ReserveOutputBuffer(
 453       dimensions, PIXEL_FORMAT_I420, frame_feedback_id, &output_buffer);
 454
 455   // Failed to reserve I420 output buffer, so drop the frame.
 456   if (reservation_result_code != ReserveResult::kSucceeded) {
 457     receiver_->OnFrameDropped(
 458         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 459     return;
 460   }
 461
 462   uint8_t* y_plane_data;
 463   uint8_t* u_plane_data;
 464   uint8_t* v_plane_data;
 465   int y_plane_stride, uv_plane_stride;
 466   GetI420BufferAccess(output_buffer, dimensions, &y_plane_data, &u_plane_data,
 467                       &v_plane_data, &y_plane_stride, &uv_plane_stride);
 468
 469   if (!buffer->Map()) {
 470     LOG(ERROR) << "Failed to map GPU memory buffer";
 471     receiver_->OnFrameDropped(
 472         VideoCaptureFrameDropReason::kGpuMemoryBufferMapFailed);
 473     return;
 474   }
 475   base::ScopedClosureRunner unmap_closure(
 476       base::BindOnce([](gfx::GpuMemoryBuffer* buffer) { buffer->Unmap(); },
 477                      base::Unretained(buffer)));
 478
 479   int ret = -EINVAL;
 480   switch (frame_format.pixel_format) {
 481     case PIXEL_FORMAT_NV12:
 482       ret = libyuv::NV12ToI420Rotate(
 483           reinterpret_cast<uint8_t*>(buffer->memory(0)), buffer->stride(0),
 484           reinterpret_cast<uint8_t*>(buffer->memory(1)), buffer->stride(1),
 485           y_plane_data, y_plane_stride, u_plane_data, uv_plane_stride,
 486           v_plane_data, uv_plane_stride, buffer->GetSize().width(),
 487           buffer->GetSize().height(), rotation_mode);
 488       break;
 489
 490     default:
 491       LOG(ERROR) << "Unsupported format: "
 492                  << VideoPixelFormatToString(frame_format.pixel_format);
 493   }
 494   if (ret) {
 495     DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
 496                   << VideoPixelFormatToString(frame_format.pixel_format);
 497     receiver_->OnFrameDropped(
 498         VideoCaptureFrameDropReason::kDeviceClientLibyuvConvertToI420Failed);
 499     return;
 500   }
 501
 502   const VideoCaptureFormat output_format = VideoCaptureFormat(
 503       dimensions, frame_format.frame_rate, PIXEL_FORMAT_I420);
 504   OnIncomingCapturedBuffer(std::move(output_buffer), output_format,
 505                            reference_time, timestamp);
 506 }
 507
 508 void VideoCaptureDeviceClient::OnIncomingCapturedExternalBuffer(
 509     CapturedExternalVideoBuffer buffer,
 510     base::TimeTicks reference_time,
 511     base::TimeDelta timestamp,
 512     const gfx::Rect& visible_rect) {
 513   ReadyFrameInBuffer ready_frame;
 514   if (CreateReadyFrameFromExternalBuffer(
 515           std::move(buffer), reference_time, timestamp, visible_rect,
 516           &ready_frame) != ReserveResult::kSucceeded) {
 517     DVLOG(2) << __func__
 518              << " CreateReadyFrameFromExternalBuffer failed: reservation "
 519                 "trakcer failed.";
 520     return;
 521   }
 522   receiver_->OnFrameReadyInBuffer(std::move(ready_frame));
 523 }
 524
 525 VideoCaptureDevice::Client::ReserveResult
 526 VideoCaptureDeviceClient::CreateReadyFrameFromExternalBuffer(
 527     CapturedExternalVideoBuffer buffer,
 528     base::TimeTicks reference_time,
 529     base::TimeDelta timestamp,
 530     const gfx::Rect& visible_rect,
 531     ReadyFrameInBuffer* ready_buffer) {
 532   // Reserve an ID for this buffer that will not conflict with any of the IDs
 533   // used by |buffer_pool_|.
 534   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
 535   int buffer_id = VideoCaptureBufferPool::kInvalidId;
 536   // Use std::move to transfer the handle ownership here since the buffer will
 537   // be created and confirm each ScopedHandle can only have one owner at a
 538   // time. Because the subsequent code mojom::VideoFrameInfoPtr needs to use the
 539   // buffer information, so here use |buffer_for_reserve_id| instead of
 540   // std::move(buffer).
 541   CapturedExternalVideoBuffer buffer_for_reserve_id =
 542       CapturedExternalVideoBuffer(std::move(buffer.handle), buffer.format,
 543                                   buffer.color_space);
 544 #if BUILDFLAG(IS_WIN)
 545   buffer_for_reserve_id.imf_buffer = std::move(buffer.imf_buffer);
 546 #endif
 547   VideoCaptureDevice::Client::ReserveResult reservation_result_code =
 548       buffer_pool_->ReserveIdForExternalBuffer(std::move(buffer_for_reserve_id),
 549                                                visible_rect.size(),
 550                                                &buffer_id_to_drop, &buffer_id);
 551   // If a buffer to retire was specified, retire one.
 552   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
 553     auto entry_iter =
 554         base::ranges::find(buffer_ids_known_by_receiver_, buffer_id_to_drop);
 555     if (entry_iter != buffer_ids_known_by_receiver_.end()) {
 556       buffer_ids_known_by_receiver_.erase(entry_iter);
 557       receiver_->OnBufferRetired(buffer_id_to_drop);
 558     }
 559   }
 560
 561   if (reservation_result_code != ReserveResult::kSucceeded) {
 562     return reservation_result_code;
 563   }
 564
 565   // Register the buffer with the receiver if it is new.
 566   if (!base::Contains(buffer_ids_known_by_receiver_, buffer_id)) {
 567     // On windows, 'GetGpuMemoryBufferHandle' will duplicate a new handle which
 568     // refers to the same object as the original handle.
 569     // https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-duplicatehandle
 570     media::mojom::VideoBufferHandlePtr buffer_handle =
 571         media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
 572             buffer_pool_->GetGpuMemoryBufferHandle(buffer_id));
 573     receiver_->OnNewBuffer(buffer_id, std::move(buffer_handle));
 574     buffer_ids_known_by_receiver_.push_back(buffer_id);
 575   }
 576
 577   // Construct the ready frame, to be passed on to the |receiver_| by the caller
 578   // of this method.
 579   mojom::VideoFrameInfoPtr info = mojom::VideoFrameInfo::New();
 580   info->timestamp = timestamp;
 581   info->pixel_format = buffer.format.pixel_format;
 582   info->color_space = buffer.color_space;
 583   info->coded_size = buffer.format.frame_size;
 584   info->visible_rect = visible_rect;
 585   info->metadata.frame_rate = buffer.format.frame_rate;
 586   info->metadata.reference_time = reference_time;
 587
 588   buffer_pool_->HoldForConsumers(buffer_id, 1);
 589   buffer_pool_->RelinquishProducerReservation(buffer_id);
 590
 591   *ready_buffer = ReadyFrameInBuffer(
 592       buffer_id, 0 /* frame_feedback_id */,
 593       std::make_unique<ScopedBufferPoolReservation<ConsumerReleaseTraits>>(
 594           buffer_pool_, buffer_id),
 595       std::move(info));
 596   return VideoCaptureDevice::Client::ReserveResult::kSucceeded;
 597 }
 598
 599 VideoCaptureDevice::Client::ReserveResult
 600 VideoCaptureDeviceClient::ReserveOutputBuffer(const gfx::Size& frame_size,
 601                                               VideoPixelFormat pixel_format,
 602                                               int frame_feedback_id,
 603                                               Buffer* buffer) {
 604   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 605   CHECK_GT(frame_size.width(), 0);
 606   CHECK_GT(frame_size.height(), 0);
 607   CHECK(IsFormatSupported(pixel_format));
 608
 609   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
 610   int buffer_id = VideoCaptureBufferPool::kInvalidId;
 611   auto reservation_result_code = buffer_pool_->ReserveForProducer(
 612       frame_size, pixel_format, nullptr, frame_feedback_id, &buffer_id,
 613       &buffer_id_to_drop);
 614   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
 615     // |buffer_pool_| has decided to release a buffer. Notify receiver in case
 616     // the buffer has already been shared with it.
 617     auto entry_iter =
 618         base::ranges::find(buffer_ids_known_by_receiver_, buffer_id_to_drop);
 619     if (entry_iter != buffer_ids_known_by_receiver_.end()) {
 620       buffer_ids_known_by_receiver_.erase(entry_iter);
 621       receiver_->OnBufferRetired(buffer_id_to_drop);
 622     }
 623   }
 624   if (reservation_result_code != ReserveResult::kSucceeded) {
 625     DVLOG(2) << __func__ << " reservation failed";
 626     return reservation_result_code;
 627   }
 628
 629   CHECK_NE(VideoCaptureBufferPool::kInvalidId, buffer_id);
 630
 631   if (!base::Contains(buffer_ids_known_by_receiver_, buffer_id)) {
 632     VideoCaptureBufferType target_buffer_type = target_buffer_type_;
 633 #if BUILDFLAG(IS_WIN) || BUILDFLAG(IS_MAC)
 634     // If MediaFoundationD3D11VideoCapture or VideoCaptureDeviceAVFoundation
 635     // fails to produce NV12 as is expected on these platforms when the target
 636     // buffer type is `kGpuMemoryBuffer`, a shared memory buffer may be sent
 637     // instead.
 638     if (target_buffer_type == VideoCaptureBufferType::kGpuMemoryBuffer &&
 639         pixel_format != PIXEL_FORMAT_NV12
 640 #if BUILDFLAG(IS_MAC)
 641         && base::FeatureList::IsEnabled(kFallbackToSharedMemoryIfNotNv12OnMac)
 642 #endif
 643     ) {
 644       target_buffer_type = VideoCaptureBufferType::kSharedMemory;
 645     }
 646 #endif
 647     media::mojom::VideoBufferHandlePtr buffer_handle;
 648     switch (target_buffer_type) {
 649       case VideoCaptureBufferType::kSharedMemory:
 650         buffer_handle = media::mojom::VideoBufferHandle::NewUnsafeShmemRegion(
 651             buffer_pool_->DuplicateAsUnsafeRegion(buffer_id));
 652         break;
 653       case VideoCaptureBufferType::kMailboxHolder:
 654         NOTREACHED();
 655         break;
 656       case VideoCaptureBufferType::kGpuMemoryBuffer:
 657         buffer_handle =
 658             media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
 659                 buffer_pool_->GetGpuMemoryBufferHandle(buffer_id));
 660         break;
 661     }
 662     receiver_->OnNewBuffer(buffer_id, std::move(buffer_handle));
 663     buffer_ids_known_by_receiver_.push_back(buffer_id);
 664   }
 665
 666   *buffer = MakeBufferStruct(buffer_pool_, buffer_id, frame_feedback_id);
 667   return ReserveResult::kSucceeded;
 668 }
 669
 670 void VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
 671     Buffer buffer,
 672     const VideoCaptureFormat& format,
 673     base::TimeTicks reference_time,
 674     base::TimeDelta timestamp) {
 675   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 676   OnIncomingCapturedBufferExt(
 677       std::move(buffer), format, gfx::ColorSpace(), reference_time, timestamp,
 678       gfx::Rect(format.frame_size), VideoFrameMetadata());
 679 }
 680
 681 void VideoCaptureDeviceClient::OnIncomingCapturedBufferExt(
 682     Buffer buffer,
 683     const VideoCaptureFormat& format,
 684     const gfx::ColorSpace& color_space,
 685     base::TimeTicks reference_time,
 686     base::TimeDelta timestamp,
 687     gfx::Rect visible_rect,
 688     const VideoFrameMetadata& additional_metadata,
 689     unsigned int encoded_data_size) {
 690   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
 691                "VideoCaptureDeviceClient::OnIncomingCapturedBufferExt");
 692   DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_);
 693
 694   VideoFrameMetadata metadata = additional_metadata;
 695   metadata.frame_rate = format.frame_rate;
 696   metadata.reference_time = reference_time;
 697
 698   mojom::VideoFrameInfoPtr info = mojom::VideoFrameInfo::New();
 699   info->timestamp = timestamp;
 700   info->pixel_format = format.pixel_format;
 701   info->color_space = color_space;
 702   info->coded_size = format.frame_size;
 703   info->visible_rect = visible_rect;
 704   info->metadata = metadata;
 705   info->is_premapped = buffer.is_premapped;
 706   info->encoded_data_size = encoded_data_size;
 707
 708   buffer_pool_->HoldForConsumers(buffer.id, 1);
 709   receiver_->OnFrameReadyInBuffer(ReadyFrameInBuffer(
 710       buffer.id, buffer.frame_feedback_id,
 711       std::make_unique<ScopedBufferPoolReservation<ConsumerReleaseTraits>>(
 712           buffer_pool_, buffer.id),
 713       std::move(info)));
 714 }
 715
 716 void VideoCaptureDeviceClient::OnError(VideoCaptureError error,
 717                                        const base::Location& from_here,
 718                                        const std::string& reason) {
 719   const std::string log_message = base::StringPrintf(
 720       "error@ %s, %s, OS message: %s", from_here.ToString().c_str(),
 721       reason.c_str(),
 722       logging::SystemErrorCodeToString(logging::GetLastSystemErrorCode())
 723           .c_str());
 724   DLOG(ERROR) << log_message;
 725   OnLog(log_message);
 726   receiver_->OnError(error);
 727 }
 728
 729 void VideoCaptureDeviceClient::OnFrameDropped(
 730     VideoCaptureFrameDropReason reason) {
 731   receiver_->OnFrameDropped(reason);
 732 }
 733
 734 void VideoCaptureDeviceClient::OnLog(const std::string& message) {
 735   receiver_->OnLog(message);
 736 }
 737
 738 void VideoCaptureDeviceClient::OnStarted() {
 739   receiver_->OnStarted();
 740 }
 741
 742 double VideoCaptureDeviceClient::GetBufferPoolUtilization() const {
 743   return buffer_pool_->GetBufferPoolUtilization();
 744 }
 745
 746 void VideoCaptureDeviceClient::OnIncomingCapturedY16Data(
 747     const uint8_t* data,
 748     int length,
 749     const VideoCaptureFormat& format,
 750     base::TimeTicks reference_time,
 751     base::TimeDelta timestamp,
 752     int frame_feedback_id) {
 753   Buffer buffer;
 754   const auto reservation_result_code = ReserveOutputBuffer(
 755       format.frame_size, PIXEL_FORMAT_Y16, frame_feedback_id, &buffer);
 756   // The input |length| can be greater than the required buffer size because of
 757   // paddings and/or alignments, but it cannot be smaller.
 758   CHECK_GE(static_cast<size_t>(length),
 759            media::VideoFrame::AllocationSize(format.pixel_format,
 760                                              format.frame_size));
 761   // Failed to reserve output buffer, so drop the frame.
 762   if (reservation_result_code != ReserveResult::kSucceeded) {
 763     receiver_->OnFrameDropped(
 764         ConvertReservationFailureToFrameDropReason(reservation_result_code));
 765     return;
 766   }
 767   auto buffer_access = buffer.handle_provider->GetHandleForInProcessAccess();
 768   memcpy(buffer_access->data(), data, length);
 769   const VideoCaptureFormat output_format = VideoCaptureFormat(
 770       format.frame_size, format.frame_rate, PIXEL_FORMAT_Y16);
 771   OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time,
 772                            timestamp);
 773 }
 774 }  // namespace media