1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
6 #define MEDIA_BASE_VIDEO_FRAME_H_
16 #include "base/check_op.h"
17 #include "base/compiler_specific.h"
18 #include "base/functional/callback.h"
19 #include "base/hash/md5.h"
20 #include "base/memory/raw_ptr.h"
21 #include "base/memory/read_only_shared_memory_region.h"
22 #include "base/memory/ref_counted.h"
23 #include "base/notreached.h"
24 #include "base/process/memory.h"
25 #include "base/synchronization/lock.h"
26 #include "base/thread_annotations.h"
27 #include "base/time/time.h"
28 #include "base/types/id_type.h"
29 #include "base/unguessable_token.h"
30 #include "build/build_config.h"
31 #include "gpu/command_buffer/common/mailbox_holder.h"
32 #include "gpu/ipc/common/vulkan_ycbcr_info.h"
33 #include "media/base/video_frame_layout.h"
34 #include "media/base/video_frame_metadata.h"
35 #include "media/base/video_types.h"
36 #include "third_party/abseil-cpp/absl/types/optional.h"
37 #include "ui/gfx/color_space.h"
38 #include "ui/gfx/geometry/rect.h"
39 #include "ui/gfx/geometry/size.h"
40 #include "ui/gfx/hdr_metadata.h"
42 #if BUILDFLAG(IS_APPLE)
43 #include <CoreVideo/CVPixelBuffer.h>
44 #include "base/apple/scoped_cftyperef.h"
45 #endif // BUILDFLAG(IS_APPLE)
47 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
48 #include "base/files/scoped_file.h"
49 #endif // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
52 class GpuMemoryBuffer;
53 struct GpuMemoryBufferHandle;
56 #if defined(TIZEN_TBM_SUPPORT)
57 #include <tbm_bufmgr.h>
58 #include <tbm_surface.h>
59 #include <tbm_surface_internal.h>
60 #include "components/viz/common/gpu/context_provider.h"
61 #include "ui/gfx/tbm_buffer_handle.h"
62 #define _DEBUG_TBM_VIDEO_RENDERING 0
63 #define _DEBUG_TBM_VIDEO_RENDERING_FPS 0
73 #if defined(TIZEN_TBM_SUPPORT)
74 constexpr auto TBM_BO_NUM_MAX = 4;
77 // Specifies the type of shared image format used by media video
78 // encoder/decoder. Currently, we have (1) one shared image (and texture)
79 // created for single planar formats eg. RGBA (2) multiple shared images created
80 // for multiplanar formats eg. P010, NV12 with one shared image for each plane
81 // eg. Y and UV passing singleplanar SharedImageFormats (kR_8, kRG_88, kR_16,
82 // etc.) and (3) one shared image created for multiplanar formats passing in
83 // legacy multiplanar SharedImageFormats that are used with external sampler.
84 // As we roll out usage of MultiPlaneFormat, this enum helps with
85 // differentiating between 3 cases: (1) SinglePlaneFormat/LegacyMultiPlaneFormat
86 // (2) MultiPlaneFormat without external sampler, and (3) MultiPlaneFormat with
87 // external sampler. NOTE: This enum is interim until all clients are converted
88 // to use MultiPlaneFormat for all multiplanar use cases; then it will be
89 // replaced with bool for external sampler usage.
90 enum class SharedImageFormatType : uint8_t {
91 // SinglePlaneFormat/LegacyMultiPlaneFormat
93 // MultiPlaneFormat without external sampler
95 // MultiPlaneFormat with external sampler
96 kSharedImageFormatExternalSampler,
99 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
101 static constexpr size_t kFrameSizeAlignment = 16;
102 static constexpr size_t kFrameSizePadding = 16;
104 static constexpr size_t kFrameAddressAlignment =
105 VideoFrameLayout::kBufferAddressAlignment;
111 kARGBPlane = kYPlane,
115 kAPlaneTriPlanar = kVPlane,
119 // These values are persisted to logs. Entries should not be renumbered and
120 // numeric values should never be reused.
121 // Defines the pixel storage type. Differentiates between directly accessible
122 // |data_| and pixels that are only indirectly accessible and not via mappable
124 // Note that VideoFrames of any StorageType can also have Texture backing,
125 // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE.
128 STORAGE_OPAQUE = 1, // We don't know how VideoFrame's pixels are stored.
129 STORAGE_UNOWNED_MEMORY = 2, // External, non owned data pointers.
130 STORAGE_OWNED_MEMORY = 3, // VideoFrame has allocated its own data buffer.
131 STORAGE_SHMEM = 4, // Backed by read-only shared memory.
132 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
133 // TODO(mcasas): Consider turning this type into STORAGE_NATIVE
134 // based on the idea of using this same enum value for both DMA
135 // buffers on Linux and CVPixelBuffers on Mac (which currently use
136 // STORAGE_UNOWNED_MEMORY) and handle it appropriately in all cases.
137 STORAGE_DMABUFS = 5, // Each plane is stored into a DmaBuf.
139 #if defined(TIZEN_TBM_SUPPORT)
140 STORAGE_TBM_SURFACE = 6,
142 #if defined(TIZEN_VIDEO_HOLE)
143 // Indicates protected media that needs to be directly rendered to hw. It
144 // is, in principle, platform independent, see http://crbug.com/323157 and
145 // https://groups.google.com/a/google.com/d/topic/chrome-gpu/eIM1RwarUmk/discussion
148 STORAGE_GPU_MEMORY_BUFFER = 8,
149 STORAGE_MAX = STORAGE_GPU_MEMORY_BUFFER,
152 // CB to be called on the mailbox backing this frame and its GpuMemoryBuffers
153 // (if they exist) when the frame is destroyed.
154 using ReleaseMailboxCB = base::OnceCallback<void(const gpu::SyncToken&)>;
155 using ReleaseMailboxAndGpuMemoryBufferCB =
156 base::OnceCallback<void(const gpu::SyncToken&,
157 std::unique_ptr<gfx::GpuMemoryBuffer>)>;
159 // Interface representing client operations on a SyncToken, i.e. insert one in
160 // the GPU Command Buffer and wait for it.
161 class SyncTokenClient {
163 SyncTokenClient() = default;
164 SyncTokenClient(const SyncTokenClient&) = delete;
165 SyncTokenClient& operator=(const SyncTokenClient&) = delete;
167 virtual void GenerateSyncToken(gpu::SyncToken* sync_token) = 0;
168 virtual void WaitSyncToken(const gpu::SyncToken& sync_token) = 0;
171 virtual ~SyncTokenClient() = default;
174 VideoFrame() = delete;
175 VideoFrame(const VideoFrame&) = delete;
176 VideoFrame& operator=(const VideoFrame&) = delete;
178 // Returns true if size is valid for a VideoFrame.
179 static bool IsValidSize(const gfx::Size& coded_size,
180 const gfx::Rect& visible_rect,
181 const gfx::Size& natural_size);
183 // Returns true if and only if the |size| is within limits, i.e., neither
184 // dimension exceeds limits::kMaxDimension and the total area doesn't exceed
185 // limits::kMaxCanvas. Prefer the overload that accepts the |coded_size|,
186 // |visible_rect|, and |natural_size| if trying to validate the VideoFrame.
187 static bool IsValidCodedSize(const gfx::Size& size);
189 // Returns true if frame configuration is valid.
190 static bool IsValidConfig(VideoPixelFormat format,
191 StorageType storage_type,
192 const gfx::Size& coded_size,
193 const gfx::Rect& visible_rect,
194 const gfx::Size& natural_size);
196 // Compute a strided layout for `format` and `coded_size`, and return a fully
197 // specified layout including offsets and plane sizes. Except that VideoFrame
198 // knows how to compute plane sizes, this method should be in
199 // `VideoFrameLayout`, probably just folded into `CreateWithStrides()`.
200 static absl::optional<VideoFrameLayout> CreateFullySpecifiedLayoutWithStrides(
201 VideoPixelFormat format,
202 const gfx::Size& coded_size);
204 // Creates a new frame in system memory with given parameters. Buffers for the
205 // frame are allocated but not initialized. The caller must not make
206 // assumptions about the actual underlying size(s), but check the returned
207 // VideoFrame instead.
208 static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format,
209 const gfx::Size& coded_size,
210 const gfx::Rect& visible_rect,
211 const gfx::Size& natural_size,
212 base::TimeDelta timestamp);
214 // Used by Chromecast only.
215 // Create a new frame that doesn't contain any valid video content. This frame
216 // is meant to be sent to compositor to inform that the compositor should
217 // punch a transparent hole so the video underlay will be visible.
218 static scoped_refptr<VideoFrame> CreateVideoHoleFrame(
219 const base::UnguessableToken& overlay_plane_id,
220 const gfx::Size& natural_size,
221 base::TimeDelta timestamp);
223 // Offers the same functionality as CreateFrame, and additionally zeroes out
224 // the initial allocated buffers.
225 static scoped_refptr<VideoFrame> CreateZeroInitializedFrame(
226 VideoPixelFormat format,
227 const gfx::Size& coded_size,
228 const gfx::Rect& visible_rect,
229 const gfx::Size& natural_size,
230 base::TimeDelta timestamp);
232 // Creates a new frame in system memory with given parameters. Buffers for the
233 // frame are allocated but not initialized. The caller should specify the
234 // physical buffer size and strides if needed in |layout| parameter.
235 static scoped_refptr<VideoFrame> CreateFrameWithLayout(
236 const VideoFrameLayout& layout,
237 const gfx::Rect& visible_rect,
238 const gfx::Size& natural_size,
239 base::TimeDelta timestamp,
240 bool zero_initialize_memory);
242 // Wraps a set of native textures with a VideoFrame.
243 // |mailbox_holders_release_cb| will be called with a sync token as the
244 // argument when the VideoFrame is to be destroyed.
245 static scoped_refptr<VideoFrame> WrapNativeTextures(
246 VideoPixelFormat format,
247 const gpu::MailboxHolder (&mailbox_holder)[kMaxPlanes],
248 ReleaseMailboxCB mailbox_holders_release_cb,
249 const gfx::Size& coded_size,
250 const gfx::Rect& visible_rect,
251 const gfx::Size& natural_size,
252 base::TimeDelta timestamp);
254 // Wraps packed image data residing in a memory buffer with a VideoFrame.
255 // The image data resides in |data| and is assumed to be packed tightly in a
256 // buffer of logical dimensions |coded_size| with the appropriate bit depth
257 // and plane count as given by |format|. Returns NULL on failure.
258 static scoped_refptr<VideoFrame> WrapExternalData(
259 VideoPixelFormat format,
260 const gfx::Size& coded_size,
261 const gfx::Rect& visible_rect,
262 const gfx::Size& natural_size,
265 base::TimeDelta timestamp,
266 size_t encoded_data_size = 0);
268 static scoped_refptr<VideoFrame> WrapExternalDataWithLayout(
269 const VideoFrameLayout& layout,
270 const gfx::Rect& visible_rect,
271 const gfx::Size& natural_size,
274 base::TimeDelta timestamp);
276 // Wraps external YUV data of the given parameters with a VideoFrame.
277 // The returned VideoFrame does not own the data passed in.
278 static scoped_refptr<VideoFrame> WrapExternalYuvData(
279 VideoPixelFormat format,
280 const gfx::Size& coded_size,
281 const gfx::Rect& visible_rect,
282 const gfx::Size& natural_size,
286 const uint8_t* y_data,
287 const uint8_t* u_data,
288 const uint8_t* v_data,
289 base::TimeDelta timestamp);
291 // Wraps external YUV data with VideoFrameLayout. The returned VideoFrame does
292 // not own the data passed in.
293 static scoped_refptr<VideoFrame> WrapExternalYuvDataWithLayout(
294 const VideoFrameLayout& layout,
295 const gfx::Rect& visible_rect,
296 const gfx::Size& natural_size,
297 const uint8_t* y_data,
298 const uint8_t* u_data,
299 const uint8_t* v_data,
300 base::TimeDelta timestamp);
302 // Wraps external YUVA data of the given parameters with a VideoFrame.
303 // The returned VideoFrame does not own the data passed in.
304 static scoped_refptr<VideoFrame> WrapExternalYuvaData(
305 VideoPixelFormat format,
306 const gfx::Size& coded_size,
307 const gfx::Rect& visible_rect,
308 const gfx::Size& natural_size,
313 const uint8_t* y_data,
314 const uint8_t* u_data,
315 const uint8_t* v_data,
316 const uint8_t* a_data,
317 base::TimeDelta timestamp);
319 // Wraps external NV12 data of the given parameters with a VideoFrame.
320 // The returned VideoFrame does not own the data passed in.
321 static scoped_refptr<VideoFrame> WrapExternalYuvData(
322 VideoPixelFormat format,
323 const gfx::Size& coded_size,
324 const gfx::Rect& visible_rect,
325 const gfx::Size& natural_size,
328 const uint8_t* y_data,
329 const uint8_t* uv_data,
330 base::TimeDelta timestamp);
332 // Wraps |gpu_memory_buffer| along with the mailboxes created from
333 // |gpu_memory_buffer|. This will transfer ownership of |gpu_memory_buffer|
334 // to the returned VideoFrame. |mailbox_holder_and_gmb_release_cb| will be
335 // called with a sync token and with |gpu_memory_buffer| as arguments when the
336 // VideoFrame is to be destroyed.
337 static scoped_refptr<VideoFrame> WrapExternalGpuMemoryBuffer(
338 const gfx::Rect& visible_rect,
339 const gfx::Size& natural_size,
340 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
341 const gpu::MailboxHolder (&mailbox_holders)[kMaxPlanes],
342 ReleaseMailboxAndGpuMemoryBufferCB mailbox_holder_and_gmb_release_cb,
343 base::TimeDelta timestamp);
345 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
346 // Wraps provided dmabufs
347 // (https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html) with a
348 // VideoFrame. The frame will take ownership of |dmabuf_fds|, and will
349 // automatically close() them on destruction. Callers can call
350 // media::DuplicateFDs() if they need to retain a copy of the FDs for
351 // themselves. Note that the FDs are consumed even in case of failure.
352 // The image data is only accessible via dmabuf fds, which are usually passed
353 // directly to a hardware device and/or to another process, or can also be
354 // mapped via mmap() for CPU access.
355 // Returns NULL on failure.
356 static scoped_refptr<VideoFrame> WrapExternalDmabufs(
357 const VideoFrameLayout& layout,
358 const gfx::Rect& visible_rect,
359 const gfx::Size& natural_size,
360 std::vector<base::ScopedFD> dmabuf_fds,
361 base::TimeDelta timestamp);
364 #if BUILDFLAG(IS_APPLE)
365 // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
366 // retained for the lifetime of the VideoFrame and released upon destruction.
367 // The image data is only accessible via the pixel buffer, which could be
368 // backed by an IOSurface from another process. All the attributes of the
369 // VideoFrame are derived from the pixel buffer, with the exception of the
370 // timestamp. If information is missing or is incompatible (for example, a
371 // pixel format that has no VideoFrame match), NULL is returned.
372 // http://crbug.com/401308
373 static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
374 CVPixelBufferRef cv_pixel_buffer,
375 base::TimeDelta timestamp);
377 // Wraps a provided IOSurface with a VideoFrame. The IOSurface is retained
378 // and locked for the lifetime of the VideoFrame. This is for unaccelerated
379 // (CPU-only) access to the IOSurface, and is not efficient. It is the path
380 // that video capture uses when hardware acceleration is disabled.
381 // https://crbug.com/1125879
382 static scoped_refptr<VideoFrame> WrapUnacceleratedIOSurface(
383 gfx::GpuMemoryBufferHandle handle,
384 const gfx::Rect& visible_rect,
385 base::TimeDelta timestamp);
388 // Wraps |frame|. |visible_rect| must be a sub rect within
389 // frame->visible_rect().
390 static scoped_refptr<VideoFrame> WrapVideoFrame(
391 scoped_refptr<VideoFrame> frame,
392 VideoPixelFormat format,
393 const gfx::Rect& visible_rect,
394 const gfx::Size& natural_size);
396 #if defined(TIZEN_TBM_SUPPORT)
397 #if BUILDFLAG(IS_TIZEN_TV)
398 static scoped_refptr<VideoFrame> WrapTBMInterProcessBuffer(
399 const gfx::Size& size,
400 base::TimeDelta timestamp,
401 gfx::TbmBufferHandle handle);
403 // Needed when we have video-frame content in tbm surface.
404 static scoped_refptr<VideoFrame> WrapTBMSurface(const gfx::Size& size,
405 base::TimeDelta timestamp,
406 gfx::TbmBufferHandle handle);
407 bool IsTBMBackend() const { return storage_type_ == STORAGE_TBM_SURFACE; }
408 unsigned GetTbmTexture() { return texture_id_; }
409 unsigned CreateTbmTextureIfNeeded(gpu::gles2::GLES2Interface* gl);
411 void SetTbmTexture(unsigned texture) { texture_id_ = texture; }
412 gfx::TbmBufferHandle GetTbmBuffer() { return buffer_handle_; }
413 unsigned GetImageID() { return image_id_; }
414 void SetImageID(unsigned image_id) { image_id_ = image_id; }
415 void SetContextProvider(
416 scoped_refptr<viz::ContextProvider> context_provider) {
417 context_provider_ = context_provider;
421 // Creates a frame which indicates end-of-stream.
422 static scoped_refptr<VideoFrame> CreateEOSFrame();
424 // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
425 static scoped_refptr<VideoFrame> CreateColorFrame(const gfx::Size& size,
429 base::TimeDelta timestamp);
431 // Allocates YV12 frame based on |size|, and sets its data to the YUV
432 // equivalent of RGB(0,0,0).
433 static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
435 // Allocates YV12A frame based on |size|, and sets its data to the YUVA
436 // equivalent of RGBA(0,0,0,0).
437 static scoped_refptr<VideoFrame> CreateTransparentFrame(
438 const gfx::Size& size);
440 #if defined(TIZEN_VIDEO_HOLE)
441 // Allocates a hole frame.
442 static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
445 static size_t NumPlanes(VideoPixelFormat format);
447 // Returns the required allocation size for a (tightly packed) frame of the
448 // given coded size and format.
449 static size_t AllocationSize(VideoPixelFormat format,
450 const gfx::Size& coded_size);
452 // Returns |dimensions| adjusted to appropriate boundaries based on |format|.
453 static gfx::Size DetermineAlignedSize(VideoPixelFormat format,
454 const gfx::Size& dimensions);
456 // Returns the plane gfx::Size (in bytes) for a plane of the given coded size
458 static gfx::Size PlaneSize(VideoPixelFormat format,
460 const gfx::Size& coded_size);
462 // Returns the plane gfx::Size (in samples) for a plane of the given coded
464 static gfx::Size PlaneSizeInSamples(VideoPixelFormat format,
466 const gfx::Size& coded_size);
468 // Returns horizontal bits per pixel for given |plane| and |format|.
469 static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane);
471 // Returns bits per pixel for given |plane| and |format|.
472 static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane);
474 // Returns the number of bytes per row for the given plane, format, and width.
475 // The width may be aligned to format requirements.
476 static size_t RowBytes(size_t plane, VideoPixelFormat format, int width);
478 // Returns the number of bytes per element for given |plane| and |format|.
479 static int BytesPerElement(VideoPixelFormat format, size_t plane);
481 // Calculates strides for each plane based on |format| and |coded_size|.
482 static std::vector<int32_t> ComputeStrides(VideoPixelFormat format,
483 const gfx::Size& coded_size);
485 // Returns the number of rows for the given plane, format, and height.
486 // The height may be aligned to format requirements.
487 static size_t Rows(size_t plane, VideoPixelFormat format, int height);
489 // Returns the number of columns for the given plane, format, and width.
490 // The width may be aligned to format requirements.
491 static size_t Columns(size_t plane, VideoPixelFormat format, int width);
493 // Used to keep a running hash of seen frames. Expects an initialized MD5
494 // context. Calls MD5Update with the context and the contents of the frame.
495 static void HashFrameForTesting(base::MD5Context* context,
496 const VideoFrame& frame);
498 // Returns true if |frame| is accesible mapped in the VideoFrame memory space.
500 static bool IsStorageTypeMappable(VideoFrame::StorageType storage_type);
502 // Returns true if |plane| is a valid plane index for the given |format|.
503 static bool IsValidPlane(VideoPixelFormat format, size_t plane);
505 // Returns the pixel size of each subsample for a given |plane| and |format|.
506 // E.g. 2x2 for the U-plane in PIXEL_FORMAT_I420.
507 static gfx::Size SampleSize(VideoPixelFormat format, size_t plane);
509 // Returns a human readable string of StorageType.
510 static std::string StorageTypeToString(VideoFrame::StorageType storage_type);
512 // A video frame wrapping external data may be backed by a read-only shared
513 // memory region. These methods are used to appropriately transform a
514 // VideoFrame created with WrapExternalData, WrapExternalYuvaData, etc. The
515 // storage type of the Video Frame will be changed to STORAGE_READ_ONLY_SHMEM.
516 // Once the backing of a VideoFrame is set, it cannot be changed.
518 // The region is NOT owned by the video frame. Both the region and its
519 // associated mapping must outlive this instance.
520 void BackWithSharedMemory(const base::ReadOnlySharedMemoryRegion* region);
522 // As above, but the VideoFrame owns the shared memory region as well as the
523 // mapping. They will be destroyed with their VideoFrame.
524 void BackWithOwnedSharedMemory(base::ReadOnlySharedMemoryRegion region,
525 base::ReadOnlySharedMemoryMapping mapping);
527 // Valid for shared memory backed VideoFrames.
528 const base::ReadOnlySharedMemoryRegion* shm_region() const {
529 DCHECK(IsValidSharedMemoryFrame());
530 DCHECK(storage_type_ == STORAGE_SHMEM);
534 // Returns true if |frame| is accessible and mapped in the VideoFrame memory
535 // space. If false, clients should refrain from accessing data(),
536 // visible_data() etc.
537 bool IsMappable() const;
539 // Returns true if |frame| has textures with any StorageType and should not be
540 // accessed via data(), visible_data() etc.
541 bool HasTextures() const;
543 // Returns the number of native textures.
544 size_t NumTextures() const;
546 // Returns true if the video frame is backed with GpuMemoryBuffer.
547 bool HasGpuMemoryBuffer() const;
549 // Gets the GpuMemoryBuffer backing the VideoFrame.
550 gfx::GpuMemoryBuffer* GetGpuMemoryBuffer() const;
552 // Returns true if the video frame was created with the given parameters.
553 bool IsSameAllocation(VideoPixelFormat format,
554 const gfx::Size& coded_size,
555 const gfx::Rect& visible_rect,
556 const gfx::Size& natural_size) const;
558 // Returns the color space of this frame's content.
559 gfx::ColorSpace ColorSpace() const;
560 void set_color_space(const gfx::ColorSpace& color_space) {
561 color_space_ = color_space;
564 const absl::optional<gfx::HDRMetadata>& hdr_metadata() const {
565 return hdr_metadata_;
568 void set_hdr_metadata(const absl::optional<gfx::HDRMetadata>& hdr_metadata) {
569 hdr_metadata_ = hdr_metadata;
572 SharedImageFormatType shared_image_format_type() const {
573 return wrapped_frame_ ? wrapped_frame_->shared_image_format_type()
574 : shared_image_format_type_;
576 void set_shared_image_format_type(SharedImageFormatType type) {
577 shared_image_format_type_ = type;
580 const VideoFrameLayout& layout() const { return layout_; }
582 VideoPixelFormat format() const { return layout_.format(); }
583 StorageType storage_type() const { return storage_type_; }
585 // Returns true if the video frame's contents should be accessed by sampling
586 // its one texture using an external sampler. Returns false if the video
587 // frame's planes should be accessed separately or if it's unknown whether an
588 // external sampler should be used.
590 // If this method returns true, VideoPixelFormatToGfxBufferFormat(format()) is
591 // guaranteed to not return nullopt.
592 // TODO(andrescj): enforce this with a test.
593 bool RequiresExternalSampler() const;
595 // The full dimensions of the video frame data.
596 const gfx::Size& coded_size() const { return layout_.coded_size(); }
597 // A subsection of [0, 0, coded_size().width(), coded_size.height()]. This
598 // can be set to "soft-apply" a cropping. It determines the pointers into
599 // the data returned by visible_data().
600 const gfx::Rect& visible_rect() const { return visible_rect_; }
601 // Specifies that the |visible_rect| section of the frame is supposed to be
602 // scaled to this size when being presented. This can be used to represent
603 // anamorphic frames, or to "soft-apply" any custom scaling.
604 const gfx::Size& natural_size() const { return natural_size_; }
606 int stride(size_t plane) const {
607 CHECK(IsValidPlane(format(), plane));
608 CHECK_LT(plane, layout_.num_planes());
609 return layout_.planes()[plane].stride;
612 // Returns the number of bytes per row and number of rows for a given plane.
614 // As opposed to stride(), row_bytes() refers to the bytes representing
615 // frame data scanlines (coded_size.width() pixels, without stride padding).
616 int row_bytes(size_t plane) const;
617 int rows(size_t plane) const;
619 // Returns the number of columns for a given plane.
620 int columns(size_t plane) const;
622 // Returns pointer to the buffer for a given plane, if this is an
623 // IsMappable() frame type. The memory is owned by VideoFrame object and must
624 // not be freed by the caller.
625 const uint8_t* data(size_t plane) const {
626 CHECK(IsValidPlane(format(), plane));
630 uint8_t* writable_data(size_t plane) {
631 // TODO(crbug.com/1435549): Also CHECK that the storage type isn't
632 // STORAGE_UNOWNED_MEMORY once non-compliant usages are fixed.
633 CHECK_NE(storage_type_, STORAGE_SHMEM);
634 CHECK(IsValidPlane(format(), plane));
636 return const_cast<uint8_t*>(data_[plane]);
639 const absl::optional<gpu::VulkanYCbCrInfo>& ycbcr_info() const {
640 return wrapped_frame_ ? wrapped_frame_->ycbcr_info() : ycbcr_info_;
643 // Returns pointer to the data in the visible region of the frame, for
644 // IsMappable() storage types. The returned pointer is offsetted into the
645 // plane buffer specified by visible_rect().origin(). Memory is owned by
646 // VideoFrame object and must not be freed by the caller.
647 const uint8_t* visible_data(size_t plane) const;
648 uint8_t* GetWritableVisibleData(size_t plane);
650 // Returns a mailbox holder for a given texture.
651 // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
652 // mailbox, the caller must wait for the included sync point.
653 const gpu::MailboxHolder& mailbox_holder(size_t texture_index) const;
655 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
656 // Returns a vector containing the backing DmaBufs for this frame. The number
657 // of returned DmaBufs will be equal or less than the number of planes of
658 // the frame. If there are less, this means that the last FD contains the
660 // Note that the returned FDs are still owned by the VideoFrame. This means
661 // that the caller shall not close them, or use them after the VideoFrame is
662 // destroyed. For such use cases, use media::DuplicateFDs() to obtain your
663 // own copy of the FDs.
664 const std::vector<base::ScopedFD>& DmabufFds() const;
666 // Returns true if |frame| has DmaBufs.
667 bool HasDmaBufs() const;
669 // Returns true if both VideoFrames are backed by DMABUF memory and point
670 // to the same set of DMABUFs, meaning that both frames use the same memory.
671 bool IsSameDmaBufsAs(const VideoFrame& frame) const;
674 #if BUILDFLAG(IS_APPLE)
675 // Returns the backing CVPixelBuffer, if present.
676 CVPixelBufferRef CvPixelBuffer() const;
679 // Sets the mailbox (and GpuMemoryBuffer, if desired) release callback.
681 // The callback may be run from ANY THREAD, and so it is up to the client to
682 // ensure thread safety.
684 // WARNING: This method is not thread safe; it should only be called if you
685 // are still the only owner of this VideoFrame.
686 void SetReleaseMailboxCB(ReleaseMailboxCB release_mailbox_cb);
687 void SetReleaseMailboxAndGpuMemoryBufferCB(
688 ReleaseMailboxAndGpuMemoryBufferCB release_mailbox_cb);
690 // Tests whether a mailbox release callback is configured.
691 bool HasReleaseMailboxCB() const;
693 // Adds a callback to be run when the VideoFrame is about to be destroyed.
694 // The callback may be run from ANY THREAD, and so it is up to the client to
695 // ensure thread safety. Although read-only access to the members of this
696 // VideoFrame is permitted while the callback executes (including
697 // VideoFrameMetadata), clients should not assume the data pointers are
699 void AddDestructionObserver(base::OnceClosure callback);
701 // Returns a dictionary of optional metadata. This contains information
702 // associated with the frame that downstream clients might use for frame-level
703 // logging, quality/performance optimizations, signaling, etc.
704 const VideoFrameMetadata& metadata() const { return metadata_; }
705 VideoFrameMetadata& metadata() { return metadata_; }
706 void set_metadata(const VideoFrameMetadata& metadata) {
707 metadata_ = metadata;
710 // Resets |metadata_|.
711 void clear_metadata() { set_metadata(VideoFrameMetadata()); }
713 // The time span between the current frame and the first frame of the stream.
714 // This is the media timestamp, and not the reference time.
715 // See VideoFrameMetadata::REFERENCE_TIME for details.
716 base::TimeDelta timestamp() const { return timestamp_; }
717 void set_timestamp(base::TimeDelta timestamp) { timestamp_ = timestamp; }
719 // It uses |client| to insert a new sync token and potentially waits on an
720 // older sync token. The final sync point will be used to release this
721 // VideoFrame. Also returns the new sync token.
722 // This method is thread safe. Both blink and compositor threads can call it.
723 gpu::SyncToken UpdateReleaseSyncToken(SyncTokenClient* client);
725 // Similar to UpdateReleaseSyncToken() but operates on the gpu::SyncToken
726 // for each plane. This should only be called when a VideoFrame has a single
727 // owner. I.e., before it has been vended after creation.
728 gpu::SyncToken UpdateMailboxHolderSyncToken(size_t plane,
729 SyncTokenClient* client);
731 // Returns a human-readable string describing |*this|.
732 std::string AsHumanReadableString() const;
734 // Unique identifier for this video frame generated at construction time. The
735 // first ID is 1. The identifier is unique within a process % overflows (which
736 // should be impossible in practice with a 64-bit unsigned integer).
738 // Note: callers may assume that ID will always correspond to a base::IdType
739 // but should not blindly assume that the underlying type will always be
740 // uint64_t (this is free to change in the future).
741 using ID = ::base::IdTypeU64<class VideoFrameIdTag>;
742 ID unique_id() const { return unique_id_; }
744 // Returns the number of bits per channel.
745 size_t BitDepth() const;
747 // Provide the sampler conversion information for the frame.
748 void set_ycbcr_info(const absl::optional<gpu::VulkanYCbCrInfo>& ycbcr_info) {
749 ycbcr_info_ = ycbcr_info;
753 friend class base::RefCountedThreadSafe<VideoFrame>;
755 enum class FrameControlType {
761 // Clients must use the static factory/wrapping methods to create a new frame.
762 // Derived classes should create their own factory/wrapping methods, and use
763 // this constructor to do basic initialization.
764 VideoFrame(const VideoFrameLayout& layout,
765 StorageType storage_type,
766 const gfx::Rect& visible_rect,
767 const gfx::Size& natural_size,
768 base::TimeDelta timestamp,
769 FrameControlType frame_control_type = FrameControlType::kNone);
770 virtual ~VideoFrame();
772 // Creates a summary of the configuration settings provided as parameters.
773 static std::string ConfigToString(const VideoPixelFormat format,
774 const VideoFrame::StorageType storage_type,
775 const gfx::Size& coded_size,
776 const gfx::Rect& visible_rect,
777 const gfx::Size& natural_size);
779 void set_data(size_t plane, const uint8_t* ptr) {
780 DCHECK(IsValidPlane(format(), plane));
786 // The constructor of VideoFrame should use IsValidConfigInternal()
787 // instead of the public IsValidConfig() to check the config, because we can
788 // create special video frames that won't pass the check by IsValidConfig().
789 static bool IsValidConfigInternal(VideoPixelFormat format,
790 FrameControlType frame_control_type,
791 const gfx::Size& coded_size,
792 const gfx::Rect& visible_rect,
793 const gfx::Size& natural_size);
795 static scoped_refptr<VideoFrame> CreateFrameInternal(
796 VideoPixelFormat format,
797 const gfx::Size& coded_size,
798 const gfx::Rect& visible_rect,
799 const gfx::Size& natural_size,
800 base::TimeDelta timestamp,
801 bool zero_initialize_memory);
803 // Return the alignment for the whole frame, calculated as the max of the
804 // alignment for each individual plane.
805 static gfx::Size CommonAlignment(VideoPixelFormat format);
807 // Tries to allocate the requisite amount of memory for this frame. Returns
808 // false if this would cause an out of memory error.
809 [[nodiscard]] bool AllocateMemory(bool zero_initialize_memory);
811 // Return plane sizes for the given layout.
813 // It first considers buffer size layout object provides. If layout's
814 // number of buffers equals to number of planes, and buffer size is assigned
815 // (non-zero), it returns buffers' size.
817 // Otherwise, it uses the first (num_buffers - 1) assigned buffers' size as
818 // plane size. Then for the rest unassigned planes, calculates their size
819 // based on format, coded size and stride for the plane.
820 static std::vector<size_t> CalculatePlaneSize(const VideoFrameLayout& layout);
822 // Calculates plane size for `layout_`.
823 std::vector<size_t> CalculatePlaneSize() const;
825 // Returns true iff the frame has a shared memory storage type, and the
826 // associated regions are valid.
827 bool IsValidSharedMemoryFrame() const;
829 template <typename T>
830 T GetVisibleDataInternal(T data, size_t plane) const;
832 // VideFrameLayout (includes format, coded_size, and strides).
833 const VideoFrameLayout layout_;
835 // Set by WrapVideoFrame to soft-apply a new set of format, visible rectangle,
836 // and natural size on |wrapped_frame_|
837 scoped_refptr<VideoFrame> wrapped_frame_;
839 // Storage type for the different planes.
840 StorageType storage_type_; // TODO(mcasas): make const
842 // Width, height, and offsets of the visible portion of the video frame. Must
843 // be a subrect of |coded_size_|. Can be odd with respect to the sample
844 // boundaries, e.g. for formats with subsampled chroma.
845 const gfx::Rect visible_rect_;
847 // Width and height of the visible portion of the video frame
848 // (|visible_rect_.size()|) with aspect ratio taken into account.
849 const gfx::Size natural_size_;
851 // Array of data pointers to each plane.
852 // TODO(mcasas): we don't know on ctor if we own |data_| or not. Change
853 // to std::unique_ptr<uint8_t, AlignedFreeDeleter> after refactoring
855 const uint8_t* data_[kMaxPlanes];
857 // Native texture mailboxes, if this is a IsTexture() frame.
858 gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
859 ReleaseMailboxAndGpuMemoryBufferCB mailbox_holders_and_gmb_release_cb_;
861 // Shared memory handle, if this frame is STORAGE_SHMEM. The region pointed
863 raw_ptr<const base::ReadOnlySharedMemoryRegion> shm_region_ = nullptr;
865 // Used if this is a STORAGE_SHMEM frame with owned shared memory.
866 // In that case, shm_region_ will refer to this region.
867 base::ReadOnlySharedMemoryRegion owned_shm_region_;
868 base::ReadOnlySharedMemoryMapping owned_shm_mapping_;
870 // GPU memory buffer, if this frame is STORAGE_GPU_MEMORY_BUFFER.
871 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer_;
873 #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
876 // Dmabufs for the frame, used when storage is STORAGE_DMABUFS. Size is either
877 // equal or less than the number of planes of the frame. If it is less, then
878 // the memory area represented by the last FD contains the remaining planes.
879 // If a STORAGE_DMABUFS frame is wrapped into another, the wrapping frame
880 // will get an extra reference to the FDs (i.e. no duplication is involved).
881 // This makes it possible to test whether two VideoFrame instances point to
882 // the same DMABUF memory by testing for
883 // (&vf1->DmabufFds() == &vf2->DmabufFds()).
884 scoped_refptr<DmabufHolder> dmabuf_fds_;
886 friend scoped_refptr<VideoFrame>
887 WrapChromeOSCompressedGpuMemoryBufferAsVideoFrame(
888 const gfx::Rect& visible_rect,
889 const gfx::Size& natural_size,
890 std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
891 base::TimeDelta timestamp);
894 #if BUILDFLAG(IS_APPLE)
895 // CVPixelBuffer, if this frame is wrapping one.
896 base::apple::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
899 base::Lock done_callbacks_lock_;
900 std::vector<base::OnceClosure> done_callbacks_
901 GUARDED_BY(done_callbacks_lock_);
903 #if defined(TIZEN_TBM_SUPPORT)
904 base::Lock tbm_lock_;
905 mutable base::Lock tbm_map_lock_;
906 gfx::TbmBufferHandle buffer_handle_;
907 unsigned texture_id_;
908 gpu::gles2::GLES2Interface* gl_;
910 scoped_refptr<viz::ContextProvider> context_provider_;
912 mutable tbm_bo bo_[TBM_BO_NUM_MAX] = {nullptr};
913 mutable tbm_bo_handle bo_handle_[TBM_BO_NUM_MAX] = {
916 mutable tbm_key key_[TBM_BO_NUM_MAX] = {0};
917 mutable uint8_t* vp_[TBM_BO_NUM_MAX] = {nullptr};
918 mutable tbm_bufmgr bufmgr_ = {nullptr};
921 base::TimeDelta timestamp_;
923 base::Lock release_sync_token_lock_;
924 gpu::SyncToken release_sync_token_ GUARDED_BY(release_sync_token_lock_);
926 VideoFrameMetadata metadata_;
928 // Generated at construction time.
931 gfx::ColorSpace color_space_;
932 absl::optional<gfx::HDRMetadata> hdr_metadata_;
934 // The format type used to create shared images. When set to Legacy creates
935 // shared images with current path; when set to SharedImageFormat with/without
936 // external sampler, creates shared image with new path (IPC) taking in
937 // SharedImageFormat with/without prefers_external_sampler set.
938 SharedImageFormatType shared_image_format_type_ =
939 SharedImageFormatType::kLegacy;
941 // Sampler conversion information which is used in vulkan context for android.
942 absl::optional<gpu::VulkanYCbCrInfo> ycbcr_info_;
944 // Allocation which makes up |data_| planes for self-allocated frames.
945 std::unique_ptr<uint8_t, base::UncheckedFreeDeleter> private_data_;
950 #endif // MEDIA_BASE_VIDEO_FRAME_H_