media/mojo/mojom/video_frame_mojom_traits.cc

   1 // Copyright 2017 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/mojo/mojom/video_frame_mojom_traits.h"
   6
   7 #include <utility>
   8 #include <vector>
   9
  10 #include "base/functional/callback_helpers.h"
  11 #include "base/logging.h"
  12 #include "base/memory/unsafe_shared_memory_region.h"
  13 #include "base/time/time.h"
  14 #include "build/build_config.h"
  15 #include "gpu/ipc/common/gpu_memory_buffer_support.h"
  16 #include "media/base/color_plane_layout.h"
  17 #include "media/base/format_utils.h"
  18 #include "media/mojo/mojom/video_frame_metadata_mojom_traits.h"
  19 #include "mojo/public/cpp/base/time_mojom_traits.h"
  20 #include "mojo/public/cpp/system/handle.h"
  21 #include "ui/gfx/mojom/buffer_types_mojom_traits.h"
  22 #include "ui/gfx/mojom/color_space_mojom_traits.h"
  23 #include "ui/gfx/mojom/hdr_metadata_mojom_traits.h"
  24
  25 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  26 #include "base/posix/eintr_wrapper.h"
  27 #endif  // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  28
  29 namespace mojo {
  30
  31 namespace {
  32
  33 base::ReadOnlySharedMemoryRegion CreateRegion(const media::VideoFrame& frame,
  34                                               std::vector<uint32_t>& offsets,
  35                                               std::vector<int32_t>& strides) {
  36   if (!media::IsYuvPlanar(frame.format()) || !media::IsOpaque(frame.format())) {
  37     DLOG(ERROR) << "format is not opaque YUV: "
  38                 << VideoPixelFormatToString(frame.format());
  39     return base::ReadOnlySharedMemoryRegion();
  40   }
  41
  42   size_t num_planes = media::VideoFrame::NumPlanes(frame.format());
  43   DCHECK_LE(num_planes, 3u);
  44   offsets.resize(num_planes);
  45   strides.resize(num_planes);
  46   if (frame.storage_type() == media::VideoFrame::STORAGE_SHMEM) {
  47     for (size_t i = 0; i < num_planes; ++i) {
  48       // This offset computation is safe because the planes are in the single
  49       // buffer, a single SharedMemoryBuffer. The first plane data must lie
  50       // in the beginning of the buffer.
  51       base::CheckedNumeric<intptr_t> offset =
  52           reinterpret_cast<intptr_t>(frame.data(i));
  53       offset -= reinterpret_cast<intptr_t>(frame.data(0));
  54       if (!offset.AssignIfValid(&offsets[i])) {
  55         DLOG(ERROR) << "Invalid offset: "
  56                     << static_cast<intptr_t>(frame.data(i) - frame.data(0));
  57         return base::ReadOnlySharedMemoryRegion();
  58       }
  59
  60       strides[i] = frame.stride(i);
  61     }
  62     return frame.shm_region()->Duplicate();
  63   }
  64
  65   // |frame| is on-memory based VideoFrame. Creates a ReadOnlySharedMemoryRegion
  66   // and copy the frame data to the region. This DCHECK is safe because of the
  67   // the conditional in a calling function.
  68   DCHECK(frame.storage_type() == media::VideoFrame::STORAGE_UNOWNED_MEMORY ||
  69          frame.storage_type() == media::VideoFrame::STORAGE_OWNED_MEMORY);
  70   std::vector<size_t> sizes(num_planes);
  71   size_t aggregate_size = 0;
  72   for (size_t i = 0; i < num_planes; ++i) {
  73     strides[i] = frame.stride(i);
  74     offsets[i] = aggregate_size;
  75     sizes[i] = media::VideoFrame::Rows(i, frame.format(),
  76                                        frame.coded_size().height()) *
  77                strides[i];
  78     aggregate_size += sizes[i];
  79   }
  80
  81   auto mapped_region = base::ReadOnlySharedMemoryRegion::Create(aggregate_size);
  82   if (!mapped_region.IsValid()) {
  83     DLOG(ERROR) << "Can't create new frame backing memory";
  84     return base::ReadOnlySharedMemoryRegion();
  85   }
  86
  87   base::WritableSharedMemoryMapping& dst_mapping = mapped_region.mapping;
  88   uint8_t* dst_data = dst_mapping.GetMemoryAs<uint8_t>();
  89   // The data from |frame| may not be consecutive between planes. Copy data into
  90   // a shared memory buffer which is tightly packed. Padding inside each planes
  91   // are preserved.
  92   for (size_t i = 0; i < num_planes; ++i) {
  93     memcpy(dst_data + offsets[i], static_cast<const void*>(frame.data(i)),
  94            sizes[i]);
  95   }
  96
  97   return std::move(mapped_region.region);
  98 }
  99
 100 media::mojom::VideoFrameDataPtr MakeVideoFrameData(
 101     const media::VideoFrame* input) {
 102   if (input->metadata().end_of_stream) {
 103     return media::mojom::VideoFrameData::NewEosData(
 104         media::mojom::EosVideoFrameData::New());
 105   }
 106
 107   if (input->storage_type() == media::VideoFrame::STORAGE_SHMEM ||
 108       input->storage_type() == media::VideoFrame::STORAGE_UNOWNED_MEMORY ||
 109       input->storage_type() == media::VideoFrame::STORAGE_OWNED_MEMORY) {
 110     std::vector<uint32_t> offsets;
 111     std::vector<int32_t> strides;
 112     auto region = CreateRegion(*input, offsets, strides);
 113     if (!region.IsValid()) {
 114       DLOG(ERROR) << "Failed to create region from VideoFrame";
 115       return nullptr;
 116     }
 117
 118     return media::mojom::VideoFrameData::NewSharedMemoryData(
 119         media::mojom::SharedMemoryVideoFrameData::New(
 120             std::move(region), std::move(strides), std::move(offsets)));
 121   }
 122
 123   std::vector<gpu::MailboxHolder> mailbox_holder(media::VideoFrame::kMaxPlanes);
 124   DCHECK_LE(input->NumTextures(), mailbox_holder.size());
 125   // STORAGE_GPU_MEMORY_BUFFER may carry meaningful or dummy mailboxes,
 126   // we should only access them when there are textures.
 127   for (size_t i = 0; i < input->NumTextures(); i++)
 128     mailbox_holder[i] = input->mailbox_holder(i);
 129
 130   if (input->storage_type() == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
 131     gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle;
 132     if (input->HasGpuMemoryBuffer())
 133       gpu_memory_buffer_handle = input->GetGpuMemoryBuffer()->CloneHandle();
 134     return media::mojom::VideoFrameData::NewGpuMemoryBufferData(
 135         media::mojom::GpuMemoryBufferVideoFrameData::New(
 136             std::move(gpu_memory_buffer_handle), std::move(mailbox_holder)));
 137   } else if (input->HasTextures()) {
 138     return media::mojom::VideoFrameData::NewMailboxData(
 139         media::mojom::MailboxVideoFrameData::New(
 140             std::move(mailbox_holder), std::move(input->ycbcr_info())));
 141   }
 142
 143   NOTREACHED() << "Unsupported VideoFrame conversion";
 144   return nullptr;
 145 }
 146
 147 }  // namespace
 148
 149 // static
 150 media::mojom::SharedImageFormatType EnumTraits<
 151     media::mojom::SharedImageFormatType,
 152     media::SharedImageFormatType>::ToMojom(media::SharedImageFormatType type) {
 153   switch (type) {
 154     case media::SharedImageFormatType::kLegacy:
 155       return media::mojom::SharedImageFormatType::kLegacy;
 156     case media::SharedImageFormatType::kSharedImageFormat:
 157       return media::mojom::SharedImageFormatType::kSharedImageFormat;
 158     case media::SharedImageFormatType::kSharedImageFormatExternalSampler:
 159       return media::mojom::SharedImageFormatType::
 160           kSharedImageFormatExternalSampler;
 161   }
 162 }
 163
 164 // static
 165 bool EnumTraits<media::mojom::SharedImageFormatType,
 166                 media::SharedImageFormatType>::
 167     FromMojom(media::mojom::SharedImageFormatType input,
 168               media::SharedImageFormatType* out) {
 169   switch (input) {
 170     case media::mojom::SharedImageFormatType::kLegacy:
 171       *out = media::SharedImageFormatType::kLegacy;
 172       return true;
 173     case media::mojom::SharedImageFormatType::kSharedImageFormat:
 174       *out = media::SharedImageFormatType::kSharedImageFormat;
 175       return true;
 176     case media::mojom::SharedImageFormatType::kSharedImageFormatExternalSampler:
 177       *out = media::SharedImageFormatType::kSharedImageFormatExternalSampler;
 178       return true;
 179   }
 180   return false;
 181 }
 182
 183 // static
 184 media::mojom::VideoFrameDataPtr StructTraits<media::mojom::VideoFrameDataView,
 185                                              scoped_refptr<media::VideoFrame>>::
 186     data(const scoped_refptr<media::VideoFrame>& input) {
 187   return media::mojom::VideoFrameDataPtr(MakeVideoFrameData(input.get()));
 188 }
 189
 190 // static
 191 bool StructTraits<media::mojom::VideoFrameDataView,
 192                   scoped_refptr<media::VideoFrame>>::
 193     Read(media::mojom::VideoFrameDataView input,
 194          scoped_refptr<media::VideoFrame>* output) {
 195   // View of the |data| member of the input media::mojom::VideoFrame.
 196   media::mojom::VideoFrameDataDataView data;
 197   input.GetDataDataView(&data);
 198
 199   if (data.is_eos_data()) {
 200     *output = media::VideoFrame::CreateEOSFrame();
 201     return !!*output;
 202   }
 203
 204   media::VideoPixelFormat format;
 205   if (!input.ReadFormat(&format))
 206     return false;
 207
 208   gfx::Size coded_size;
 209   if (!input.ReadCodedSize(&coded_size))
 210     return false;
 211
 212   gfx::Rect visible_rect;
 213   if (!input.ReadVisibleRect(&visible_rect))
 214     return false;
 215
 216   if (!gfx::Rect(coded_size).Contains(visible_rect))
 217     return false;
 218
 219   gfx::Size natural_size;
 220   if (!input.ReadNaturalSize(&natural_size))
 221     return false;
 222
 223   base::TimeDelta timestamp;
 224   if (!input.ReadTimestamp(&timestamp))
 225     return false;
 226
 227   scoped_refptr<media::VideoFrame> frame;
 228   if (data.is_shared_memory_data()) {
 229     media::mojom::SharedMemoryVideoFrameDataDataView shared_memory_data;
 230     data.GetSharedMemoryDataDataView(&shared_memory_data);
 231
 232     base::ReadOnlySharedMemoryRegion region;
 233     if (!shared_memory_data.ReadFrameData(&region))
 234       return false;
 235
 236     mojo::ArrayDataView<uint32_t> offsets;
 237     shared_memory_data.GetOffsetsDataView(&offsets);
 238
 239     mojo::ArrayDataView<int32_t> strides;
 240     shared_memory_data.GetStridesDataView(&strides);
 241
 242     base::ReadOnlySharedMemoryMapping mapping = region.Map();
 243     if (!mapping.IsValid()) {
 244       DLOG(ERROR) << "Failed to map ReadOnlySharedMemoryRegion";
 245       return false;
 246     }
 247
 248     const size_t num_planes = offsets.size();
 249     if (num_planes == 0 || num_planes > 3) {
 250       DLOG(ERROR) << "Invalid number of planes: " << num_planes;
 251       return false;
 252     }
 253
 254     uint8_t* addr[3] = {};
 255     std::vector<media::ColorPlaneLayout> planes(num_planes);
 256     for (size_t i = 0; i < num_planes; i++) {
 257       addr[i] =
 258           const_cast<uint8_t*>(mapping.GetMemoryAs<uint8_t>()) + offsets[i];
 259       planes[i].stride = strides[i];
 260       planes[i].offset = base::strict_cast<size_t>(offsets[i]);
 261       planes[i].size = i + 1 < num_planes
 262                            ? offsets[i + 1] - offsets[i]
 263                            : mapping.size() - offsets[num_planes - 1];
 264     }
 265
 266     auto layout = media::VideoFrameLayout::CreateWithPlanes(format, coded_size,
 267                                                             std::move(planes));
 268     if (!layout || !layout->FitsInContiguousBufferOfSize(mapping.size())) {
 269       DLOG(ERROR) << "Invalid layout";
 270       return false;
 271     }
 272
 273     frame = media::VideoFrame::WrapExternalYuvDataWithLayout(
 274         *layout, visible_rect, natural_size, addr[0], addr[1], addr[2],
 275         timestamp);
 276     if (frame) {
 277       frame->BackWithOwnedSharedMemory(std::move(region), std::move(mapping));
 278     }
 279   } else if (data.is_gpu_memory_buffer_data()) {
 280     media::mojom::GpuMemoryBufferVideoFrameDataDataView gpu_memory_buffer_data;
 281     data.GetGpuMemoryBufferDataDataView(&gpu_memory_buffer_data);
 282
 283     gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle;
 284     if (!gpu_memory_buffer_data.ReadGpuMemoryBufferHandle(
 285             &gpu_memory_buffer_handle)) {
 286       DLOG(ERROR) << "Failed to read GpuMemoryBufferHandle";
 287       return false;
 288     }
 289
 290     std::vector<gpu::MailboxHolder> mailbox_holder;
 291     if (!gpu_memory_buffer_data.ReadMailboxHolder(&mailbox_holder)) {
 292       DLOG(WARNING) << "Failed to get mailbox holder";
 293     }
 294     if (mailbox_holder.size() > media::VideoFrame::kMaxPlanes) {
 295       DLOG(ERROR) << "The size of mailbox holder is too large: "
 296                   << mailbox_holder.size();
 297       return false;
 298     }
 299
 300     gpu::MailboxHolder mailbox_holder_array[media::VideoFrame::kMaxPlanes];
 301     for (size_t i = 0; i < mailbox_holder.size(); i++)
 302       mailbox_holder_array[i] = mailbox_holder[i];
 303
 304     absl::optional<gfx::BufferFormat> buffer_format =
 305         VideoPixelFormatToGfxBufferFormat(format);
 306     if (!buffer_format)
 307       return false;
 308
 309     // Shared memory GMBs do not support VEA/CAMERA usage.
 310     const gfx::BufferUsage buffer_usage =
 311         (gpu_memory_buffer_handle.type ==
 312          gfx::GpuMemoryBufferType::SHARED_MEMORY_BUFFER)
 313             ? gfx::BufferUsage::SCANOUT_CPU_READ_WRITE
 314             : gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE;
 315
 316     gpu::GpuMemoryBufferSupport support;
 317     std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer =
 318         support.CreateGpuMemoryBufferImplFromHandle(
 319             std::move(gpu_memory_buffer_handle), coded_size, *buffer_format,
 320             buffer_usage, base::NullCallback());
 321     if (!gpu_memory_buffer)
 322       return false;
 323
 324     frame = media::VideoFrame::WrapExternalGpuMemoryBuffer(
 325         visible_rect, natural_size, std::move(gpu_memory_buffer),
 326         mailbox_holder_array, base::NullCallback(), timestamp);
 327   } else if (data.is_mailbox_data()) {
 328     media::mojom::MailboxVideoFrameDataDataView mailbox_data;
 329     data.GetMailboxDataDataView(&mailbox_data);
 330
 331     std::vector<gpu::MailboxHolder> mailbox_holder;
 332     if (!mailbox_data.ReadMailboxHolder(&mailbox_holder))
 333       return false;
 334
 335     gpu::MailboxHolder mailbox_holder_array[media::VideoFrame::kMaxPlanes];
 336     for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; i++)
 337       mailbox_holder_array[i] = mailbox_holder[i];
 338
 339     absl::optional<gpu::VulkanYCbCrInfo> ycbcr_info;
 340     if (!mailbox_data.ReadYcbcrData(&ycbcr_info))
 341       return false;
 342
 343     frame = media::VideoFrame::WrapNativeTextures(
 344         format, mailbox_holder_array, media::VideoFrame::ReleaseMailboxCB(),
 345         coded_size, visible_rect, natural_size, timestamp);
 346     frame->set_ycbcr_info(ycbcr_info);
 347   } else {
 348     // TODO(sandersd): Switch on the union tag to avoid this ugliness?
 349     NOTREACHED_NORETURN();
 350   }
 351
 352   if (!frame) {
 353     return false;
 354   }
 355
 356   media::VideoFrameMetadata metadata;
 357   if (!input.ReadMetadata(&metadata))
 358     return false;
 359
 360   frame->set_metadata(metadata);
 361
 362   gfx::ColorSpace color_space;
 363   if (!input.ReadColorSpace(&color_space))
 364     return false;
 365   frame->set_color_space(color_space);
 366
 367   absl::optional<gfx::HDRMetadata> hdr_metadata;
 368   if (!input.ReadHdrMetadata(&hdr_metadata))
 369     return false;
 370   frame->set_hdr_metadata(std::move(hdr_metadata));
 371
 372   media::SharedImageFormatType shared_image_format_type;
 373   if (!input.ReadSharedImageFormatType(&shared_image_format_type)) {
 374     return false;
 375   }
 376   frame->set_shared_image_format_type(shared_image_format_type);
 377
 378   *output = std::move(frame);
 379   return true;
 380 }
 381
 382 }  // namespace mojo