media/renderers/video_frame_yuv_mailboxes_holder.cc

   1 // Copyright 2021 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/renderers/video_frame_yuv_mailboxes_holder.h"
   6
   7 #include <GLES3/gl3.h>
   8
   9 #include "base/logging.h"
  10 #include "components/viz/common/gpu/raster_context_provider.h"
  11 #include "components/viz/common/resources/shared_image_format.h"
  12 #include "components/viz/common/resources/shared_image_format_utils.h"
  13 #include "gpu/GLES2/gl2extchromium.h"
  14 #include "gpu/command_buffer/client/raster_interface.h"
  15 #include "gpu/command_buffer/client/shared_image_interface.h"
  16 #include "gpu/command_buffer/common/shared_image_usage.h"
  17 #include "media/base/media_switches.h"
  18 #include "third_party/skia/include/core/SkColorSpace.h"
  19 #include "third_party/skia/include/core/SkImage.h"
  20 #include "third_party/skia/include/core/SkSurface.h"
  21 #include "third_party/skia/include/core/SkYUVAPixmaps.h"
  22 #include "third_party/skia/include/gpu/GrDirectContext.h"
  23 #include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h"
  24 #include "third_party/skia/include/gpu/ganesh/SkSurfaceGanesh.h"
  25 #include "third_party/skia/include/gpu/ganesh/gl/GrGLBackendSurface.h"
  26 #include "third_party/skia/include/gpu/gl/GrGLTypes.h"
  27
  28 namespace media {
  29
  30 namespace {
  31
  32 viz::SharedImageFormat PlaneSharedImageFormat(int num_channels,
  33                                               bool supports_red) {
  34   switch (num_channels) {
  35     case 1:
  36       return supports_red ? viz::SinglePlaneFormat::kR_8
  37                           : viz::SinglePlaneFormat::kLUMINANCE_8;
  38     case 2:
  39       return viz::SinglePlaneFormat::kRG_88;
  40     case 3:
  41       return viz::SinglePlaneFormat::kRGBX_8888;
  42     case 4:
  43       return viz::SinglePlaneFormat::kRGBA_8888;
  44   }
  45   NOTREACHED_NORETURN();
  46 }
  47
  48 // Returns multiplanar format equivalent of a VideoPixelFormat.
  49 viz::SharedImageFormat VideoPixelFormatToSharedImageFormat(
  50     VideoPixelFormat video_format) {
  51   switch (video_format) {
  52     case PIXEL_FORMAT_NV12:
  53       return viz::MultiPlaneFormat::kNV12;
  54     case PIXEL_FORMAT_P016LE:
  55       return viz::MultiPlaneFormat::kP010;
  56     case PIXEL_FORMAT_NV12A:
  57       return viz::MultiPlaneFormat::kNV12A;
  58     case PIXEL_FORMAT_I420:
  59       return viz::MultiPlaneFormat::kI420;
  60     case PIXEL_FORMAT_I420A:
  61       return viz::MultiPlaneFormat::kI420;
  62     default:
  63       NOTREACHED_NORETURN();
  64   }
  65 }
  66
  67 GLenum PlaneGLFormat(int num_channels,
  68                      viz::RasterContextProvider* context_provider) {
  69   return context_provider->GetGrGLTextureFormat(PlaneSharedImageFormat(
  70       num_channels, context_provider->ContextCapabilities().texture_rg));
  71 }
  72
  73 }  // namespace
  74
  75 VideoFrameYUVMailboxesHolder::VideoFrameYUVMailboxesHolder() = default;
  76
  77 VideoFrameYUVMailboxesHolder::~VideoFrameYUVMailboxesHolder() {
  78   ReleaseCachedData();
  79 }
  80
  81 void VideoFrameYUVMailboxesHolder::ReleaseCachedData() {
  82   if (holders_[0].mailbox.IsZero())
  83     return;
  84
  85   ReleaseTextures();
  86
  87   // Don't destroy shared images we don't own.
  88   if (!created_shared_images_)
  89     return;
  90
  91   auto* ri = provider_->RasterInterface();
  92   DCHECK(ri);
  93   gpu::SyncToken token;
  94   ri->GenUnverifiedSyncTokenCHROMIUM(token.GetData());
  95
  96   auto* sii = provider_->SharedImageInterface();
  97   DCHECK(sii);
  98   for (auto& mailbox_holder : holders_) {
  99     if (!mailbox_holder.mailbox.IsZero())
 100       sii->DestroySharedImage(token, mailbox_holder.mailbox);
 101     mailbox_holder.mailbox.SetZero();
 102   }
 103
 104   created_shared_images_ = false;
 105 }
 106
 107 void VideoFrameYUVMailboxesHolder::VideoFrameToMailboxes(
 108     const VideoFrame* video_frame,
 109     viz::RasterContextProvider* raster_context_provider,
 110     gpu::Mailbox mailboxes[SkYUVAInfo::kMaxPlanes],
 111     bool allow_multiplanar_for_upload) {
 112   yuva_info_ = VideoFrameGetSkYUVAInfo(video_frame);
 113   num_planes_ = yuva_info_.planeDimensions(plane_sizes_);
 114
 115   // If we have cached shared images but the provider or video has changed we
 116   // need to release shared images created on the old context and recreate them.
 117   if (created_shared_images_ &&
 118       (provider_.get() != raster_context_provider ||
 119        video_frame->coded_size() != cached_video_size_ ||
 120        video_frame->ColorSpace() != cached_video_color_space_)) {
 121     ReleaseCachedData();
 122   }
 123   provider_ = raster_context_provider;
 124   DCHECK(provider_);
 125   auto* ri = provider_->RasterInterface();
 126   DCHECK(ri);
 127
 128   if (video_frame->HasTextures()) {
 129     // Video frames with mailboxes will have shared images per plane as new
 130     // multiplanar shared image with mailbox path should not go through
 131     // VideoFrameToMailboxes.
 132     DCHECK_EQ(num_planes_, video_frame->NumTextures());
 133     for (size_t plane = 0; plane < video_frame->NumTextures(); ++plane) {
 134       holders_[plane] = video_frame->mailbox_holder(plane);
 135       DCHECK(holders_[plane].texture_target == GL_TEXTURE_2D ||
 136              holders_[plane].texture_target == GL_TEXTURE_EXTERNAL_OES ||
 137              holders_[plane].texture_target == GL_TEXTURE_RECTANGLE_ARB)
 138           << "Unsupported texture target " << std::hex << std::showbase
 139           << holders_[plane].texture_target;
 140       ri->WaitSyncTokenCHROMIUM(holders_[plane].sync_token.GetConstData());
 141       mailboxes[plane] = holders_[plane].mailbox;
 142     }
 143     return;
 144   }
 145
 146   CHECK(!video_frame->HasTextures());
 147   constexpr SkAlphaType kPlaneAlphaType = kPremul_SkAlphaType;
 148   auto* sii = provider_->SharedImageInterface();
 149   DCHECK(sii);
 150   uint32_t mailbox_usage;
 151   auto& caps = provider_->ContextCapabilities();
 152   if (caps.supports_oop_raster) {
 153     mailbox_usage = gpu::SHARED_IMAGE_USAGE_RASTER |
 154                     gpu::SHARED_IMAGE_USAGE_OOP_RASTERIZATION;
 155   } else {
 156     mailbox_usage = gpu::SHARED_IMAGE_USAGE_GLES2;
 157   }
 158
 159   // Enabled with flags UseWritePixelsYUV and
 160   // UseMultiPlaneFormatForHardwareVideo.
 161   if (allow_multiplanar_for_upload) {
 162     SkPixmap pixmaps[SkYUVAInfo::kMaxPlanes] = {};
 163     viz::SharedImageFormat format =
 164         VideoPixelFormatToSharedImageFormat(video_frame->format());
 165     CHECK(format.is_multi_plane());
 166
 167     // Create a multiplanar shared image to upload the data to, if one doesn't
 168     // exist already.
 169     if (!created_shared_images_) {
 170       holders_[0].mailbox = sii->CreateSharedImage(
 171           format, video_frame->coded_size(), video_frame->ColorSpace(),
 172           kTopLeft_GrSurfaceOrigin, kPlaneAlphaType, mailbox_usage,
 173           "VideoFrameYUV", gpu::kNullSurfaceHandle);
 174       holders_[0].texture_target = GL_TEXTURE_2D;
 175
 176       // Split up shared image creation from upload so we only have to wait on
 177       // one sync token.
 178       ri->WaitSyncTokenCHROMIUM(sii->GenUnverifiedSyncToken().GetConstData());
 179
 180       cached_video_size_ = video_frame->coded_size();
 181       cached_video_color_space_ = video_frame->ColorSpace();
 182       created_shared_images_ = true;
 183     }
 184
 185     for (size_t plane = 0; plane < num_planes_; ++plane) {
 186       SkColorType color_type =
 187           viz::ToClosestSkColorType(/*gpu_compositing=*/true, format, plane);
 188       SkImageInfo info =
 189           SkImageInfo::Make(plane_sizes_[plane], color_type, kPlaneAlphaType);
 190       pixmaps[plane] =
 191           SkPixmap(info, video_frame->data(plane), video_frame->stride(plane));
 192     }
 193     SkYUVAPixmaps yuv_pixmap =
 194         SkYUVAPixmaps::FromExternalPixmaps(yuva_info_, pixmaps);
 195     ri->WritePixelsYUV(holders_[0].mailbox, yuv_pixmap);
 196     mailboxes[0] = holders_[0].mailbox;
 197     return;
 198   }
 199
 200   // Create shared images to upload the data to, if they doesn't exist already.
 201   if (!created_shared_images_) {
 202     for (size_t plane = 0; plane < num_planes_; ++plane) {
 203       gfx::Size tex_size = {plane_sizes_[plane].width(),
 204                             plane_sizes_[plane].height()};
 205       int num_channels = yuva_info_.numChannelsInPlane(plane);
 206       viz::SharedImageFormat format =
 207           PlaneSharedImageFormat(num_channels, caps.texture_rg);
 208       holders_[plane].mailbox = sii->CreateSharedImage(
 209           format, tex_size, video_frame->ColorSpace(), kTopLeft_GrSurfaceOrigin,
 210           kPlaneAlphaType, mailbox_usage, "VideoFrameYUV",
 211           gpu::kNullSurfaceHandle);
 212       holders_[plane].texture_target = GL_TEXTURE_2D;
 213     }
 214
 215     // Split up shared image creation from upload so we only have to wait on
 216     // one sync token.
 217     ri->WaitSyncTokenCHROMIUM(sii->GenUnverifiedSyncToken().GetConstData());
 218
 219     cached_video_size_ = video_frame->coded_size();
 220     cached_video_color_space_ = video_frame->ColorSpace();
 221     created_shared_images_ = true;
 222   }
 223
 224   // If we have cached shared images that have been imported release them to
 225   // prevent writing to a shared image for which we're holding read access.
 226   ReleaseTextures();
 227
 228   for (size_t plane = 0; plane < num_planes_; ++plane) {
 229     int num_channels = yuva_info_.numChannelsInPlane(plane);
 230     SkColorType color_type = SkYUVAPixmapInfo::DefaultColorTypeForDataType(
 231         SkYUVAPixmaps::DataType::kUnorm8, num_channels);
 232     SkImageInfo info =
 233         SkImageInfo::Make(plane_sizes_[plane], color_type, kPlaneAlphaType);
 234     ri->WritePixels(
 235         holders_[plane].mailbox, /*dst_x_offset=*/0,
 236         /*dst_y_offset=*/0, /*dst_plane_index=*/0, GL_TEXTURE_2D,
 237         SkPixmap(info, video_frame->data(plane), video_frame->stride(plane)));
 238     mailboxes[plane] = holders_[plane].mailbox;
 239   }
 240 }
 241
 242 GrYUVABackendTextures VideoFrameYUVMailboxesHolder::VideoFrameToSkiaTextures(
 243     const VideoFrame* video_frame,
 244     viz::RasterContextProvider* raster_context_provider,
 245     bool for_surface) {
 246   gpu::Mailbox mailboxes[kMaxPlanes];
 247   VideoFrameToMailboxes(video_frame, raster_context_provider, mailboxes,
 248                         /*allow_multiplanar_for_upload=*/false);
 249   ImportTextures(for_surface);
 250   GrBackendTexture backend_textures[SkYUVAInfo::kMaxPlanes];
 251   for (size_t plane = 0; plane < num_planes_; ++plane) {
 252     backend_textures[plane] = GrBackendTextures::MakeGL(
 253         plane_sizes_[plane].width(), plane_sizes_[plane].height(),
 254         skgpu::Mipmapped::kNo, textures_[plane].texture);
 255   }
 256   return GrYUVABackendTextures(yuva_info_, backend_textures,
 257                                kTopLeft_GrSurfaceOrigin);
 258 }
 259
 260 sk_sp<SkImage> VideoFrameYUVMailboxesHolder::VideoFrameToSkImage(
 261     const VideoFrame* video_frame,
 262     viz::RasterContextProvider* raster_context_provider,
 263     sk_sp<SkColorSpace> reinterpret_color_space) {
 264   GrDirectContext* gr_context = raster_context_provider->GrContext();
 265   DCHECK(gr_context);
 266
 267   GrYUVABackendTextures yuva_backend_textures = VideoFrameToSkiaTextures(
 268       video_frame, raster_context_provider, /*for_surface=*/false);
 269   auto rgb_color_space =
 270       reinterpret_color_space
 271           ? reinterpret_color_space
 272           : video_frame->ColorSpace().GetAsFullRangeRGB().ToSkColorSpace();
 273
 274   DCHECK(yuva_backend_textures.isValid());
 275   auto result = SkImages::TextureFromYUVATextures(
 276       gr_context, yuva_backend_textures, rgb_color_space);
 277   DCHECK(result);
 278   return result;
 279 }
 280
 281 bool VideoFrameYUVMailboxesHolder::VideoFrameToPlaneSkSurfaces(
 282     const VideoFrame* video_frame,
 283     viz::RasterContextProvider* raster_context_provider,
 284     sk_sp<SkSurface> surfaces[SkYUVAInfo::kMaxPlanes]) {
 285   for (size_t plane = 0; plane < SkYUVAInfo::kMaxPlanes; ++plane)
 286     surfaces[plane] = nullptr;
 287
 288   if (!video_frame->HasTextures()) {
 289     // The below call to VideoFrameToSkiaTextures would blit |video_frame| into
 290     // a temporary SharedImage, which would be exposed as a SkSurface. That is
 291     // probably undesirable (it has no current use cases), so just return an
 292     // error.
 293     DLOG(ERROR) << "VideoFrameToPlaneSkSurfaces requires texture backing.";
 294     return false;
 295   }
 296
 297   GrDirectContext* gr_context = raster_context_provider->GrContext();
 298   DCHECK(gr_context);
 299   GrYUVABackendTextures yuva_backend_textures = VideoFrameToSkiaTextures(
 300       video_frame, raster_context_provider, /*for_surface=*/true);
 301
 302   bool result = true;
 303   for (size_t plane = 0; plane < num_planes_; ++plane) {
 304     const int num_channels = yuva_info_.numChannelsInPlane(plane);
 305     SkColorType color_type = SkYUVAPixmapInfo::DefaultColorTypeForDataType(
 306         SkYUVAPixmaps::DataType::kUnorm8, num_channels);
 307     // Gray is not renderable.
 308     if (color_type == kGray_8_SkColorType)
 309       color_type = kAlpha_8_SkColorType;
 310
 311     auto surface = SkSurfaces::WrapBackendTexture(
 312         gr_context, yuva_backend_textures.texture(plane),
 313         kTopLeft_GrSurfaceOrigin, /*sampleCnt=*/1, color_type,
 314         SkColorSpace::MakeSRGB(), nullptr);
 315     if (!surface) {
 316       DLOG(ERROR)
 317           << "VideoFrameToPlaneSkSurfaces failed to make surface for plane "
 318           << plane << " of " << num_planes_ << ".";
 319       result = false;
 320     }
 321     surfaces[plane] = surface;
 322   }
 323   return result;
 324 }
 325
 326 void VideoFrameYUVMailboxesHolder::ImportTextures(bool for_surface) {
 327   DCHECK(!imported_textures_)
 328       << "Textures should always be released after converting video frame. "
 329          "Call ReleaseTextures() for each call to VideoFrameToSkiaTextures()";
 330
 331   auto* ri = provider_->RasterInterface();
 332   for (size_t plane = 0; plane < num_planes_; ++plane) {
 333     textures_[plane].texture.fID =
 334         ri->CreateAndConsumeForGpuRaster(holders_[plane].mailbox);
 335     if (holders_[plane].mailbox.IsSharedImage()) {
 336       textures_[plane].is_shared_image = true;
 337       ri->BeginSharedImageAccessDirectCHROMIUM(
 338           textures_[plane].texture.fID,
 339           for_surface ? GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM
 340                       : GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM);
 341     } else {
 342       textures_[plane].is_shared_image = false;
 343     }
 344
 345     int num_channels = yuva_info_.numChannelsInPlane(plane);
 346     textures_[plane].texture.fTarget = holders_[plane].texture_target;
 347     textures_[plane].texture.fFormat =
 348         PlaneGLFormat(num_channels, provider_.get());
 349   }
 350
 351   imported_textures_ = true;
 352 }
 353
 354 void VideoFrameYUVMailboxesHolder::ReleaseTextures() {
 355   if (!imported_textures_)
 356     return;
 357
 358   auto* ri = provider_->RasterInterface();
 359   DCHECK(ri);
 360   for (auto& tex_info : textures_) {
 361     if (!tex_info.texture.fID)
 362       continue;
 363
 364     if (tex_info.is_shared_image)
 365       ri->EndSharedImageAccessDirectCHROMIUM(tex_info.texture.fID);
 366     ri->DeleteGpuRasterTexture(tex_info.texture.fID);
 367
 368     tex_info.texture.fID = 0;
 369   }
 370
 371   imported_textures_ = false;
 372 }
 373
 374 // static
 375 std::tuple<SkYUVAInfo::PlaneConfig, SkYUVAInfo::Subsampling>
 376 VideoFrameYUVMailboxesHolder::VideoPixelFormatToSkiaValues(
 377     VideoPixelFormat video_format) {
 378   // To expand support for additional VideoFormats expand this switch. Note that
 379   // we do assume 8 bit formats. With that exception, anything else should work.
 380   switch (video_format) {
 381     case PIXEL_FORMAT_NV12:
 382     case PIXEL_FORMAT_P016LE:
 383       return {SkYUVAInfo::PlaneConfig::kY_UV, SkYUVAInfo::Subsampling::k420};
 384     case PIXEL_FORMAT_NV12A:
 385       return {SkYUVAInfo::PlaneConfig::kY_UV_A, SkYUVAInfo::Subsampling::k420};
 386     case PIXEL_FORMAT_I420:
 387       return {SkYUVAInfo::PlaneConfig::kY_U_V, SkYUVAInfo::Subsampling::k420};
 388     case PIXEL_FORMAT_I420A:
 389       return {SkYUVAInfo::PlaneConfig::kY_U_V_A, SkYUVAInfo::Subsampling::k420};
 390     default:
 391       return {SkYUVAInfo::PlaneConfig::kUnknown,
 392               SkYUVAInfo::Subsampling::kUnknown};
 393   }
 394 }
 395
 396 // static
 397 SkYUVAInfo VideoFrameYUVMailboxesHolder::VideoFrameGetSkYUVAInfo(
 398     const VideoFrame* video_frame) {
 399   SkISize video_size{video_frame->coded_size().width(),
 400                      video_frame->coded_size().height()};
 401   auto plane_config = SkYUVAInfo::PlaneConfig::kUnknown;
 402   auto subsampling = SkYUVAInfo::Subsampling::kUnknown;
 403   std::tie(plane_config, subsampling) =
 404       VideoPixelFormatToSkiaValues(video_frame->format());
 405
 406   // TODO(crbug.com/828599): This should really default to rec709.
 407   SkYUVColorSpace color_space = kRec601_SkYUVColorSpace;
 408   video_frame->ColorSpace().ToSkYUVColorSpace(video_frame->BitDepth(),
 409                                               &color_space);
 410   return SkYUVAInfo(video_size, plane_config, subsampling, color_space);
 411 }
 412
 413 }  // namespace media