media/renderers/video_frame_rgba_to_yuva_converter.cc

   1 // Copyright 2021 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/renderers/video_frame_rgba_to_yuva_converter.h"
   6
   7 #include "base/logging.h"
   8 #include "base/memory/ptr_util.h"
   9 #include "base/memory/raw_ptr.h"
  10 #include "components/viz/common/gpu/raster_context_provider.h"
  11 #include "components/viz/common/resources/shared_image_format_utils.h"
  12 #include "gpu/command_buffer/client/raster_interface.h"
  13 #include "gpu/command_buffer/client/shared_image_interface.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "gpu/command_buffer/common/shared_image_capabilities.h"
  16 #include "media/base/simple_sync_token_client.h"
  17 #include "media/base/wait_and_replace_sync_token_client.h"
  18 #include "media/renderers/video_frame_yuv_converter.h"
  19 #include "media/renderers/video_frame_yuv_mailboxes_holder.h"
  20 #include "skia/ext/rgba_to_yuva.h"
  21 #include "third_party/skia/include/core/SkColorSpace.h"
  22 #include "third_party/skia/include/core/SkImage.h"
  23 #include "third_party/skia/include/gpu/GrDirectContext.h"
  24 #include "third_party/skia/include/gpu/GrTypes.h"
  25 #include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h"
  26 #include "third_party/skia/include/gpu/ganesh/gl/GrGLBackendSurface.h"
  27 #include "third_party/skia/include/gpu/gl/GrGLTypes.h"
  28 #include "ui/gfx/gpu_memory_buffer.h"
  29
  30 namespace {
  31
  32 // Given a gpu::MailboxHolder and a viz::RasterContextProvider, create scoped
  33 // access to the texture as an SkImage.
  34 class ScopedAcceleratedSkImage {
  35  public:
  36   static std::unique_ptr<ScopedAcceleratedSkImage> Create(
  37       viz::RasterContextProvider* provider,
  38       viz::SharedImageFormat format,
  39       const gfx::Size& size,
  40       const gfx::ColorSpace& color_space,
  41       GrSurfaceOrigin surface_origin,
  42       const gpu::MailboxHolder& mailbox_holder) {
  43     auto* ri = provider->RasterInterface();
  44     DCHECK(ri);
  45     GrDirectContext* gr_context = provider->GrContext();
  46     DCHECK(gr_context);
  47
  48     if (!mailbox_holder.mailbox.IsSharedImage()) {
  49       DLOG(ERROR) << "Cannot created SkImage for non-SharedImage mailbox.";
  50       return nullptr;
  51     }
  52
  53     ri->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
  54
  55     uint32_t texture_id =
  56         ri->CreateAndConsumeForGpuRaster(mailbox_holder.mailbox);
  57     if (!texture_id) {
  58       DLOG(ERROR) << "Failed to create texture for mailbox.";
  59       return nullptr;
  60     }
  61     ri->BeginSharedImageAccessDirectCHROMIUM(
  62         texture_id, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
  63
  64     GrGLTextureInfo gl_info = {
  65         mailbox_holder.texture_target,
  66         texture_id,
  67         provider->GetGrGLTextureFormat(format),
  68     };
  69     auto backend_texture = GrBackendTextures::MakeGL(
  70         size.width(), size.height(), skgpu::Mipmapped::kNo, gl_info);
  71
  72     SkColorType color_type = viz::ToClosestSkColorType(
  73         /*gpu_compositing=*/true, format);
  74     sk_sp<SkImage> sk_image = SkImages::BorrowTextureFrom(
  75         gr_context, backend_texture, surface_origin, color_type,
  76         kOpaque_SkAlphaType, color_space.ToSkColorSpace());
  77     if (!sk_image) {
  78       DLOG(ERROR) << "Failed to SkImage for StaticBitmapImage.";
  79       ri->EndSharedImageAccessDirectCHROMIUM(texture_id);
  80       ri->DeleteGpuRasterTexture(texture_id);
  81       return nullptr;
  82     }
  83
  84     return base::WrapUnique<ScopedAcceleratedSkImage>(
  85         new ScopedAcceleratedSkImage(provider, texture_id,
  86                                      std::move(sk_image)));
  87   }
  88
  89   ~ScopedAcceleratedSkImage() {
  90     auto* ri = provider_->RasterInterface();
  91     DCHECK(ri);
  92     GrDirectContext* gr_context = provider_->GrContext();
  93     DCHECK(gr_context);
  94
  95     sk_image_ = nullptr;
  96     if (texture_id_) {
  97       ri->EndSharedImageAccessDirectCHROMIUM(texture_id_);
  98       ri->DeleteGpuRasterTexture(texture_id_);
  99     }
 100   }
 101
 102   sk_sp<SkImage> sk_image() { return sk_image_; }
 103
 104  private:
 105   ScopedAcceleratedSkImage(viz::RasterContextProvider* provider,
 106                            uint32_t texture_id,
 107                            sk_sp<SkImage> sk_image)
 108       : provider_(provider), texture_id_(texture_id), sk_image_(sk_image) {}
 109
 110   const raw_ptr<viz::RasterContextProvider> provider_;
 111   uint32_t texture_id_ = 0;
 112   sk_sp<SkImage> sk_image_;
 113 };
 114
 115 }  // namespace
 116
 117 namespace media {
 118
 119 bool CopyRGBATextureToVideoFrame(viz::RasterContextProvider* provider,
 120                                  viz::SharedImageFormat src_format,
 121                                  const gfx::Size& src_size,
 122                                  const gfx::ColorSpace& src_color_space,
 123                                  GrSurfaceOrigin src_surface_origin,
 124                                  const gpu::MailboxHolder& src_mailbox_holder,
 125                                  VideoFrame* dst_video_frame) {
 126   DCHECK_EQ(dst_video_frame->format(), PIXEL_FORMAT_NV12);
 127
 128   auto* ri = provider->RasterInterface();
 129   DCHECK(ri);
 130
 131   // If context is lost for any reason e.g. creating shared image failed, we
 132   // cannot distinguish between OOP and non-OOP raster based on GrContext().
 133   if (ri->GetGraphicsResetStatusKHR() != GL_NO_ERROR) {
 134     DLOG(ERROR) << "Raster context lost.";
 135     return false;
 136   }
 137
 138   // With OOP raster, if RGB->YUV conversion is unsupported, the CopySharedImage
 139   // calls will fail on the service side with no ability to detect failure on
 140   // the client side. Check for support here and early out if it's unsupported.
 141   if (!provider->GrContext() &&
 142       !provider->ContextCapabilities().supports_yuv_rgb_conversion) {
 143     DVLOG(1) << "RGB->YUV conversion not supported";
 144     return false;
 145   }
 146
 147 #if BUILDFLAG(IS_WIN)
 148   // CopyToGpuMemoryBuffer is only supported for D3D shared images on Windows.
 149   if (!provider->SharedImageInterface()->GetCapabilities().shared_image_d3d) {
 150     DVLOG(1) << "CopyToGpuMemoryBuffer not supported.";
 151     return false;
 152   }
 153 #endif  // BUILDFLAG(IS_WIN)
 154
 155   if (provider->ContextCapabilities().supports_yuv_rgb_conversion &&
 156       src_mailbox_holder.mailbox.IsSharedImage()) {
 157     ri->WaitSyncTokenCHROMIUM(src_mailbox_holder.sync_token.GetConstData());
 158     if (dst_video_frame->shared_image_format_type() ==
 159         SharedImageFormatType::kLegacy) {
 160       SkYUVAInfo yuva_info =
 161           VideoFrameYUVMailboxesHolder::VideoFrameGetSkYUVAInfo(
 162               dst_video_frame);
 163       gpu::Mailbox yuva_mailboxes[SkYUVAInfo::kMaxPlanes];
 164       for (int plane = 0; plane < yuva_info.numPlanes(); ++plane) {
 165         gpu::MailboxHolder dst_mailbox_holder =
 166             dst_video_frame->mailbox_holder(plane);
 167         ri->WaitSyncTokenCHROMIUM(dst_mailbox_holder.sync_token.GetConstData());
 168         yuva_mailboxes[plane] = dst_mailbox_holder.mailbox;
 169       }
 170       ri->ConvertRGBAToYUVAMailboxes(
 171           yuva_info.yuvColorSpace(), yuva_info.planeConfig(),
 172           yuva_info.subsampling(), yuva_mailboxes, src_mailbox_holder.mailbox);
 173     } else {
 174       gpu::MailboxHolder dst_mailbox_holder =
 175           dst_video_frame->mailbox_holder(0);
 176       ri->WaitSyncTokenCHROMIUM(dst_mailbox_holder.sync_token.GetConstData());
 177
 178       // `unpack_flip_y` should be set if the surface origin of the source
 179       // doesn't match that of the destination, which is created with
 180       // kTopLeft_GrSurfaceOrigin.
 181       // TODO(crbug.com/1453515): If this codepath is used with destinations
 182       // that are created with other surface origins, will need to generalize
 183       // this.
 184       bool unpack_flip_y = (src_surface_origin != kTopLeft_GrSurfaceOrigin);
 185
 186       // Note: the destination video frame can have a coded size that is larger
 187       // than that of the source video to account for alignment needs. In this
 188       // case, both this codepath and the the legacy codepath above stretch to
 189       // fill the destination. Cropping would clearly be more correct, but
 190       // implementing that behavior in CopySharedImage() for the MultiplanarSI
 191       // case resulted in pixeltest failures due to pixel bleeding around image
 192       // borders that we weren't able to resolve (see crbug.com/1451025 for
 193       // details).
 194       // TODO(crbug.com/1451025): Update this comment when we resolve that bug
 195       // and change CopySharedImage() to crop rather than stretch.
 196       ri->CopySharedImage(src_mailbox_holder.mailbox,
 197                           dst_mailbox_holder.mailbox, GL_TEXTURE_2D, 0, 0, 0, 0,
 198                           src_size.width(), src_size.height(), unpack_flip_y,
 199                           /*unpack_premultiply_alpha=*/false);
 200     }
 201   } else {
 202     // We shouldn't be here with OOP-raster since supports_yuv_rgb_conversion
 203     // should be true in that case. We can end up here with non-OOP raster when
 204     // dealing with legacy mailbox when YUV-RGB conversion is unsupported by GL.
 205     CHECK(provider->GrContext());
 206     // Create an accelerated SkImage for the source.
 207     auto scoped_sk_image = ScopedAcceleratedSkImage::Create(
 208         provider, src_format, src_size, src_color_space, src_surface_origin,
 209         src_mailbox_holder);
 210     if (!scoped_sk_image) {
 211       DLOG(ERROR) << "Failed to create accelerated SkImage for RGBA to YUVA "
 212                      "conversion.";
 213       return false;
 214     }
 215
 216     // Create SkSurfaces for the destination planes.
 217     sk_sp<SkSurface> sk_surfaces[SkYUVAInfo::kMaxPlanes];
 218     SkSurface* sk_surface_ptrs[SkYUVAInfo::kMaxPlanes] = {nullptr};
 219     VideoFrameYUVMailboxesHolder holder;
 220     if (!holder.VideoFrameToPlaneSkSurfaces(dst_video_frame, provider,
 221                                             sk_surfaces)) {
 222       DLOG(ERROR) << "Failed to create SkSurfaces for VideoFrame.";
 223       return false;
 224     }
 225
 226     // Make GrContext wait for `dst_video_frame`. Waiting on the mailbox tokens
 227     // here ensures that all writes are completed in cases where the underlying
 228     // GpuMemoryBuffer and SharedImage resources have been reused.
 229     ri->Flush();
 230     WaitAndReplaceSyncTokenClient client(ri);
 231     for (int plane = 0; plane < holder.yuva_info().numPlanes(); ++plane) {
 232       sk_surface_ptrs[plane] = sk_surfaces[plane].get();
 233       dst_video_frame->UpdateMailboxHolderSyncToken(plane, &client);
 234     }
 235
 236     // Do the blit.
 237     skia::BlitRGBAToYUVA(scoped_sk_image->sk_image().get(), sk_surface_ptrs,
 238                          holder.yuva_info());
 239     provider->GrContext()->flushAndSubmit(GrSyncCpu::kNo);
 240   }
 241   ri->Flush();
 242
 243 #if BUILDFLAG(IS_WIN)
 244   // For shared memory GMBs on Windows we needed to explicitly request a copy
 245   // from the shared image GPU texture to the GMB.
 246   DCHECK(dst_video_frame->HasGpuMemoryBuffer());
 247   DCHECK_EQ(dst_video_frame->GetGpuMemoryBuffer()->GetType(),
 248             gfx::SHARED_MEMORY_BUFFER);
 249
 250   gpu::SyncToken blit_done_sync_token;
 251   ri->GenUnverifiedSyncTokenCHROMIUM(blit_done_sync_token.GetData());
 252
 253   auto* sii = provider->SharedImageInterface();
 254   for (size_t plane = 0; plane < dst_video_frame->NumTextures(); ++plane) {
 255     const auto& mailbox = dst_video_frame->mailbox_holder(plane).mailbox;
 256     sii->CopyToGpuMemoryBuffer(blit_done_sync_token, mailbox);
 257   }
 258
 259   // Synchronize RasterInterface with SharedImageInterface. We want to generate
 260   // the final SyncToken from the RasterInterface since callers might be using
 261   // RasterInterface::Finish() to ensure synchronization in cases where
 262   // SignalSyncToken can't be used (e.g. webrtc video frame adapter).
 263   auto copy_to_gmb_done_sync_token = sii->GenUnverifiedSyncToken();
 264   ri->WaitSyncTokenCHROMIUM(copy_to_gmb_done_sync_token.GetData());
 265 #endif  // BUILDFLAG(IS_WIN)
 266
 267   // Make access to the `dst_video_frame` wait on copy completion. We also
 268   // update the ReleaseSyncToken here since it's used when the underlying
 269   // GpuMemoryBuffer and SharedImage resources are returned to the pool.
 270   gpu::SyncToken completion_sync_token;
 271   ri->GenSyncTokenCHROMIUM(completion_sync_token.GetData());
 272   SimpleSyncTokenClient simple_client(completion_sync_token);
 273   for (size_t plane = 0; plane < dst_video_frame->NumTextures(); ++plane) {
 274     dst_video_frame->UpdateMailboxHolderSyncToken(plane, &simple_client);
 275   }
 276   dst_video_frame->UpdateReleaseSyncToken(&simple_client);
 277   return true;
 278 }
 279
 280 }  // namespace media