1 // Copyright 2017 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/mojo/mojom/video_frame_mojom_traits.h"
10 #include "base/functional/callback_helpers.h"
11 #include "base/logging.h"
12 #include "base/memory/unsafe_shared_memory_region.h"
13 #include "base/time/time.h"
14 #include "build/build_config.h"
15 #include "gpu/ipc/common/gpu_memory_buffer_support.h"
16 #include "media/base/color_plane_layout.h"
17 #include "media/base/format_utils.h"
18 #include "media/mojo/mojom/video_frame_metadata_mojom_traits.h"
19 #include "mojo/public/cpp/base/time_mojom_traits.h"
20 #include "mojo/public/cpp/system/handle.h"
21 #include "ui/gfx/mojom/buffer_types_mojom_traits.h"
22 #include "ui/gfx/mojom/color_space_mojom_traits.h"
23 #include "ui/gfx/mojom/hdr_metadata_mojom_traits.h"
25 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
26 #include "base/posix/eintr_wrapper.h"
27 #endif // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
33 base::ReadOnlySharedMemoryRegion CreateRegion(const media::VideoFrame& frame,
34 std::vector<uint32_t>& offsets,
35 std::vector<int32_t>& strides) {
36 if (!media::IsYuvPlanar(frame.format()) || !media::IsOpaque(frame.format())) {
37 DLOG(ERROR) << "format is not opaque YUV: "
38 << VideoPixelFormatToString(frame.format());
39 return base::ReadOnlySharedMemoryRegion();
42 size_t num_planes = media::VideoFrame::NumPlanes(frame.format());
43 DCHECK_LE(num_planes, 3u);
44 offsets.resize(num_planes);
45 strides.resize(num_planes);
46 if (frame.storage_type() == media::VideoFrame::STORAGE_SHMEM) {
47 for (size_t i = 0; i < num_planes; ++i) {
48 // This offset computation is safe because the planes are in the single
49 // buffer, a single SharedMemoryBuffer. The first plane data must lie
50 // in the beginning of the buffer.
51 base::CheckedNumeric<intptr_t> offset =
52 reinterpret_cast<intptr_t>(frame.data(i));
53 offset -= reinterpret_cast<intptr_t>(frame.data(0));
54 if (!offset.AssignIfValid(&offsets[i])) {
55 DLOG(ERROR) << "Invalid offset: "
56 << static_cast<intptr_t>(frame.data(i) - frame.data(0));
57 return base::ReadOnlySharedMemoryRegion();
60 strides[i] = frame.stride(i);
62 return frame.shm_region()->Duplicate();
65 // |frame| is on-memory based VideoFrame. Creates a ReadOnlySharedMemoryRegion
66 // and copy the frame data to the region. This DCHECK is safe because of the
67 // the conditional in a calling function.
68 DCHECK(frame.storage_type() == media::VideoFrame::STORAGE_UNOWNED_MEMORY ||
69 frame.storage_type() == media::VideoFrame::STORAGE_OWNED_MEMORY);
70 std::vector<size_t> sizes(num_planes);
71 size_t aggregate_size = 0;
72 for (size_t i = 0; i < num_planes; ++i) {
73 strides[i] = frame.stride(i);
74 offsets[i] = aggregate_size;
75 sizes[i] = media::VideoFrame::Rows(i, frame.format(),
76 frame.coded_size().height()) *
78 aggregate_size += sizes[i];
81 auto mapped_region = base::ReadOnlySharedMemoryRegion::Create(aggregate_size);
82 if (!mapped_region.IsValid()) {
83 DLOG(ERROR) << "Can't create new frame backing memory";
84 return base::ReadOnlySharedMemoryRegion();
87 base::WritableSharedMemoryMapping& dst_mapping = mapped_region.mapping;
88 uint8_t* dst_data = dst_mapping.GetMemoryAs<uint8_t>();
89 // The data from |frame| may not be consecutive between planes. Copy data into
90 // a shared memory buffer which is tightly packed. Padding inside each planes
92 for (size_t i = 0; i < num_planes; ++i) {
93 memcpy(dst_data + offsets[i], static_cast<const void*>(frame.data(i)),
97 return std::move(mapped_region.region);
100 media::mojom::VideoFrameDataPtr MakeVideoFrameData(
101 const media::VideoFrame* input) {
102 if (input->metadata().end_of_stream) {
103 return media::mojom::VideoFrameData::NewEosData(
104 media::mojom::EosVideoFrameData::New());
107 if (input->storage_type() == media::VideoFrame::STORAGE_SHMEM ||
108 input->storage_type() == media::VideoFrame::STORAGE_UNOWNED_MEMORY ||
109 input->storage_type() == media::VideoFrame::STORAGE_OWNED_MEMORY) {
110 std::vector<uint32_t> offsets;
111 std::vector<int32_t> strides;
112 auto region = CreateRegion(*input, offsets, strides);
113 if (!region.IsValid()) {
114 DLOG(ERROR) << "Failed to create region from VideoFrame";
118 return media::mojom::VideoFrameData::NewSharedMemoryData(
119 media::mojom::SharedMemoryVideoFrameData::New(
120 std::move(region), std::move(strides), std::move(offsets)));
123 std::vector<gpu::MailboxHolder> mailbox_holder(media::VideoFrame::kMaxPlanes);
124 DCHECK_LE(input->NumTextures(), mailbox_holder.size());
125 // STORAGE_GPU_MEMORY_BUFFER may carry meaningful or dummy mailboxes,
126 // we should only access them when there are textures.
127 for (size_t i = 0; i < input->NumTextures(); i++)
128 mailbox_holder[i] = input->mailbox_holder(i);
130 if (input->storage_type() == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
131 gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle;
132 if (input->HasGpuMemoryBuffer())
133 gpu_memory_buffer_handle = input->GetGpuMemoryBuffer()->CloneHandle();
134 return media::mojom::VideoFrameData::NewGpuMemoryBufferData(
135 media::mojom::GpuMemoryBufferVideoFrameData::New(
136 std::move(gpu_memory_buffer_handle), std::move(mailbox_holder)));
137 } else if (input->HasTextures()) {
138 return media::mojom::VideoFrameData::NewMailboxData(
139 media::mojom::MailboxVideoFrameData::New(
140 std::move(mailbox_holder), std::move(input->ycbcr_info())));
143 NOTREACHED() << "Unsupported VideoFrame conversion";
150 media::mojom::SharedImageFormatType EnumTraits<
151 media::mojom::SharedImageFormatType,
152 media::SharedImageFormatType>::ToMojom(media::SharedImageFormatType type) {
154 case media::SharedImageFormatType::kLegacy:
155 return media::mojom::SharedImageFormatType::kLegacy;
156 case media::SharedImageFormatType::kSharedImageFormat:
157 return media::mojom::SharedImageFormatType::kSharedImageFormat;
158 case media::SharedImageFormatType::kSharedImageFormatExternalSampler:
159 return media::mojom::SharedImageFormatType::
160 kSharedImageFormatExternalSampler;
165 bool EnumTraits<media::mojom::SharedImageFormatType,
166 media::SharedImageFormatType>::
167 FromMojom(media::mojom::SharedImageFormatType input,
168 media::SharedImageFormatType* out) {
170 case media::mojom::SharedImageFormatType::kLegacy:
171 *out = media::SharedImageFormatType::kLegacy;
173 case media::mojom::SharedImageFormatType::kSharedImageFormat:
174 *out = media::SharedImageFormatType::kSharedImageFormat;
176 case media::mojom::SharedImageFormatType::kSharedImageFormatExternalSampler:
177 *out = media::SharedImageFormatType::kSharedImageFormatExternalSampler;
184 media::mojom::VideoFrameDataPtr StructTraits<media::mojom::VideoFrameDataView,
185 scoped_refptr<media::VideoFrame>>::
186 data(const scoped_refptr<media::VideoFrame>& input) {
187 return media::mojom::VideoFrameDataPtr(MakeVideoFrameData(input.get()));
191 bool StructTraits<media::mojom::VideoFrameDataView,
192 scoped_refptr<media::VideoFrame>>::
193 Read(media::mojom::VideoFrameDataView input,
194 scoped_refptr<media::VideoFrame>* output) {
195 // View of the |data| member of the input media::mojom::VideoFrame.
196 media::mojom::VideoFrameDataDataView data;
197 input.GetDataDataView(&data);
199 if (data.is_eos_data()) {
200 *output = media::VideoFrame::CreateEOSFrame();
204 media::VideoPixelFormat format;
205 if (!input.ReadFormat(&format))
208 gfx::Size coded_size;
209 if (!input.ReadCodedSize(&coded_size))
212 gfx::Rect visible_rect;
213 if (!input.ReadVisibleRect(&visible_rect))
216 if (!gfx::Rect(coded_size).Contains(visible_rect))
219 gfx::Size natural_size;
220 if (!input.ReadNaturalSize(&natural_size))
223 base::TimeDelta timestamp;
224 if (!input.ReadTimestamp(×tamp))
227 scoped_refptr<media::VideoFrame> frame;
228 if (data.is_shared_memory_data()) {
229 media::mojom::SharedMemoryVideoFrameDataDataView shared_memory_data;
230 data.GetSharedMemoryDataDataView(&shared_memory_data);
232 base::ReadOnlySharedMemoryRegion region;
233 if (!shared_memory_data.ReadFrameData(®ion))
236 mojo::ArrayDataView<uint32_t> offsets;
237 shared_memory_data.GetOffsetsDataView(&offsets);
239 mojo::ArrayDataView<int32_t> strides;
240 shared_memory_data.GetStridesDataView(&strides);
242 base::ReadOnlySharedMemoryMapping mapping = region.Map();
243 if (!mapping.IsValid()) {
244 DLOG(ERROR) << "Failed to map ReadOnlySharedMemoryRegion";
248 const size_t num_planes = offsets.size();
249 if (num_planes == 0 || num_planes > 3) {
250 DLOG(ERROR) << "Invalid number of planes: " << num_planes;
254 uint8_t* addr[3] = {};
255 std::vector<media::ColorPlaneLayout> planes(num_planes);
256 for (size_t i = 0; i < num_planes; i++) {
258 const_cast<uint8_t*>(mapping.GetMemoryAs<uint8_t>()) + offsets[i];
259 planes[i].stride = strides[i];
260 planes[i].offset = base::strict_cast<size_t>(offsets[i]);
261 planes[i].size = i + 1 < num_planes
262 ? offsets[i + 1] - offsets[i]
263 : mapping.size() - offsets[num_planes - 1];
266 auto layout = media::VideoFrameLayout::CreateWithPlanes(format, coded_size,
268 if (!layout || !layout->FitsInContiguousBufferOfSize(mapping.size())) {
269 DLOG(ERROR) << "Invalid layout";
273 frame = media::VideoFrame::WrapExternalYuvDataWithLayout(
274 *layout, visible_rect, natural_size, addr[0], addr[1], addr[2],
277 frame->BackWithOwnedSharedMemory(std::move(region), std::move(mapping));
279 } else if (data.is_gpu_memory_buffer_data()) {
280 media::mojom::GpuMemoryBufferVideoFrameDataDataView gpu_memory_buffer_data;
281 data.GetGpuMemoryBufferDataDataView(&gpu_memory_buffer_data);
283 gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle;
284 if (!gpu_memory_buffer_data.ReadGpuMemoryBufferHandle(
285 &gpu_memory_buffer_handle)) {
286 DLOG(ERROR) << "Failed to read GpuMemoryBufferHandle";
290 std::vector<gpu::MailboxHolder> mailbox_holder;
291 if (!gpu_memory_buffer_data.ReadMailboxHolder(&mailbox_holder)) {
292 DLOG(WARNING) << "Failed to get mailbox holder";
294 if (mailbox_holder.size() > media::VideoFrame::kMaxPlanes) {
295 DLOG(ERROR) << "The size of mailbox holder is too large: "
296 << mailbox_holder.size();
300 gpu::MailboxHolder mailbox_holder_array[media::VideoFrame::kMaxPlanes];
301 for (size_t i = 0; i < mailbox_holder.size(); i++)
302 mailbox_holder_array[i] = mailbox_holder[i];
304 absl::optional<gfx::BufferFormat> buffer_format =
305 VideoPixelFormatToGfxBufferFormat(format);
309 // Shared memory GMBs do not support VEA/CAMERA usage.
310 const gfx::BufferUsage buffer_usage =
311 (gpu_memory_buffer_handle.type ==
312 gfx::GpuMemoryBufferType::SHARED_MEMORY_BUFFER)
313 ? gfx::BufferUsage::SCANOUT_CPU_READ_WRITE
314 : gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE;
316 gpu::GpuMemoryBufferSupport support;
317 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer =
318 support.CreateGpuMemoryBufferImplFromHandle(
319 std::move(gpu_memory_buffer_handle), coded_size, *buffer_format,
320 buffer_usage, base::NullCallback());
321 if (!gpu_memory_buffer)
324 frame = media::VideoFrame::WrapExternalGpuMemoryBuffer(
325 visible_rect, natural_size, std::move(gpu_memory_buffer),
326 mailbox_holder_array, base::NullCallback(), timestamp);
327 } else if (data.is_mailbox_data()) {
328 media::mojom::MailboxVideoFrameDataDataView mailbox_data;
329 data.GetMailboxDataDataView(&mailbox_data);
331 std::vector<gpu::MailboxHolder> mailbox_holder;
332 if (!mailbox_data.ReadMailboxHolder(&mailbox_holder))
335 gpu::MailboxHolder mailbox_holder_array[media::VideoFrame::kMaxPlanes];
336 for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; i++)
337 mailbox_holder_array[i] = mailbox_holder[i];
339 absl::optional<gpu::VulkanYCbCrInfo> ycbcr_info;
340 if (!mailbox_data.ReadYcbcrData(&ycbcr_info))
343 frame = media::VideoFrame::WrapNativeTextures(
344 format, mailbox_holder_array, media::VideoFrame::ReleaseMailboxCB(),
345 coded_size, visible_rect, natural_size, timestamp);
346 frame->set_ycbcr_info(ycbcr_info);
348 // TODO(sandersd): Switch on the union tag to avoid this ugliness?
349 NOTREACHED_NORETURN();
356 media::VideoFrameMetadata metadata;
357 if (!input.ReadMetadata(&metadata))
360 frame->set_metadata(metadata);
362 gfx::ColorSpace color_space;
363 if (!input.ReadColorSpace(&color_space))
365 frame->set_color_space(color_space);
367 absl::optional<gfx::HDRMetadata> hdr_metadata;
368 if (!input.ReadHdrMetadata(&hdr_metadata))
370 frame->set_hdr_metadata(std::move(hdr_metadata));
372 media::SharedImageFormatType shared_image_format_type;
373 if (!input.ReadSharedImageFormatType(&shared_image_format_type)) {
376 frame->set_shared_image_format_type(shared_image_format_type);
378 *output = std::move(frame);