1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_
6 #define MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_
11 #include "base/memory/scoped_refptr.h"
12 #include "base/sequence_checker.h"
13 #include "base/threading/thread_checker.h"
14 #include "base/timer/timer.h"
15 #include "cc/paint/paint_canvas.h"
16 #include "cc/paint/paint_flags.h"
17 #include "cc/paint/paint_image.h"
18 #include "components/viz/common/resources/shared_image_format.h"
19 #include "gpu/command_buffer/common/mailbox.h"
20 #include "media/base/media_export.h"
21 #include "media/base/timestamp_constants.h"
22 #include "media/base/video_frame.h"
23 #include "media/base/video_transformation.h"
24 #include "media/renderers/video_frame_yuv_converter.h"
25 #include "third_party/abseil-cpp/absl/types/optional.h"
40 class RasterContextProvider;
44 class VideoTextureBacking;
46 // Handles rendering of VideoFrames to PaintCanvases.
47 class MEDIA_EXPORT PaintCanvasVideoRenderer {
49 // Specifies the chroma upsampling filter used for pixel formats with chroma
50 // subsampling (YUV 4:2:0 and YUV 4:2:2).
52 // NOTE: Keep the numeric values in sync with libyuv::FilterMode.
54 kFilterNone = 0, // Nearest neighbor.
55 kFilterBilinear = 2, // Bilinear interpolation.
58 PaintCanvasVideoRenderer();
60 PaintCanvasVideoRenderer(const PaintCanvasVideoRenderer&) = delete;
61 PaintCanvasVideoRenderer& operator=(const PaintCanvasVideoRenderer&) = delete;
63 ~PaintCanvasVideoRenderer();
65 // Paints |video_frame| translated and scaled to |dest_rect| on |canvas|.
67 // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d|
68 // and |context_support| must be provided.
70 // If |video_frame| is nullptr or an unsupported format, |dest_rect| will be
72 void Paint(scoped_refptr<VideoFrame> video_frame,
73 cc::PaintCanvas* canvas,
74 const gfx::RectF& dest_rect,
75 cc::PaintFlags& flags,
76 VideoTransformation video_transformation,
77 viz::RasterContextProvider* raster_context_provider);
79 // Paints |video_frame|, scaled to its |video_frame->visible_rect().size()|
80 // on |canvas|. Note that the origin of |video_frame->visible_rect()| is
81 // ignored -- the copy is done to the origin of |canvas|.
83 // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d|
84 // and |context_support| must be provided.
85 void Copy(scoped_refptr<VideoFrame> video_frame,
86 cc::PaintCanvas* canvas,
87 viz::RasterContextProvider* raster_context_provider);
89 // Convert the contents of |video_frame| to raw RGB pixels. |rgb_pixels|
90 // should point into a buffer large enough to hold as many 32 bit RGBA pixels
91 // as are in the visible_rect() area of the frame. |premultiply_alpha|
92 // indicates whether the R, G, B samples in |rgb_pixels| should be multiplied
93 // by alpha. |filter| specifies the chroma upsampling filter used for pixel
94 // formats with chroma subsampling. If chroma planes in the pixel format are
95 // not subsampled, |filter| is ignored. |disable_threading| indicates whether
96 // this method should convert |video_frame| without posting any tasks to
97 // base::ThreadPool, regardless of the frame size. If this method is called
98 // from a task running in base::ThreadPool, setting |disable_threading| to
99 // true can avoid a potential temporary deadlock of base::ThreadPool. See
100 // crbug.com/1402841.
102 // NOTE: If |video_frame| doesn't have an alpha plane, all the A samples in
103 // |rgb_pixels| will be 255 (equivalent to an alpha of 1.0) and therefore the
104 // value of |premultiply_alpha| has no effect on the R, G, B samples in
106 static void ConvertVideoFrameToRGBPixels(const media::VideoFrame* video_frame,
109 bool premultiply_alpha = true,
110 FilterMode filter = kFilterNone,
111 bool disable_threading = false);
113 // The output format that ConvertVideoFrameToRGBPixels will write.
114 static viz::SharedImageFormat GetRGBPixelsOutputFormat();
116 // Copy the contents of |video_frame| to |texture| of |destination_gl|.
118 // The format of |video_frame| must be VideoFrame::NATIVE_TEXTURE.
119 bool CopyVideoFrameTexturesToGLTexture(
120 viz::RasterContextProvider* raster_context_provider,
121 gpu::gles2::GLES2Interface* destination_gl,
122 const gpu::Capabilities& destination_gl_capabilities,
123 scoped_refptr<VideoFrame> video_frame,
125 unsigned int texture,
126 unsigned int internal_format,
130 bool premultiply_alpha,
133 // Copy the CPU-side YUV contents of |video_frame| to texture |texture| in
134 // context |destination_gl|.
135 // |level|, |internal_format|, |type| specify target texture |texture|.
136 // The format of |video_frame| must be mappable.
137 // |context_3d| has a GrContext that may be used during the copy.
138 // CorrectLastImageDimensions() ensures that the source texture will be
139 // cropped to |visible_rect|. Returns true on success.
140 bool CopyVideoFrameYUVDataToGLTexture(
141 viz::RasterContextProvider* raster_context_provider,
142 gpu::gles2::GLES2Interface* destination_gl,
143 const gpu::Capabilities& destination_gl_capabilities,
144 scoped_refptr<VideoFrame> video_frame,
146 unsigned int texture,
147 unsigned int internal_format,
151 bool premultiply_alpha,
154 // Calls texImage2D where the texture image data source is the contents of
155 // |video_frame|. Texture |texture| needs to be created and bound to |target|
156 // before this call and the binding is active upon return.
157 // This is an optimization of WebGL |video_frame| TexImage2D implementation
158 // for specific combinations of |video_frame| and |texture| formats; e.g. if
159 // |frame format| is Y16, optimizes conversion of normalized 16-bit content
160 // and calls texImage2D to |texture|. |level|, |internal_format|, |format| and
161 // |type| are WebGL texImage2D parameters.
162 // Returns false if there is no implementation for given parameters.
163 static bool TexImage2D(unsigned target,
165 gpu::gles2::GLES2Interface* gl,
166 const gpu::Capabilities& gpu_capabilities,
167 VideoFrame* video_frame,
173 bool premultiply_alpha);
175 // Calls texSubImage2D where the texture image data source is the contents of
177 // This is an optimization of WebGL |video_frame| TexSubImage2D implementation
178 // for specific combinations of |video_frame| and texture |format| and |type|;
179 // e.g. if |frame format| is Y16, converts unsigned 16-bit value to target
180 // |format| and calls WebGL texSubImage2D. |level|, |format|, |type|,
181 // |xoffset| and |yoffset| are texSubImage2D parameters.
182 // Returns false if there is no implementation for given parameters.
183 static bool TexSubImage2D(unsigned target,
184 gpu::gles2::GLES2Interface* gl,
185 VideoFrame* video_frame,
192 bool premultiply_alpha);
194 // Copies VideoFrame contents to the `destination` shared image. if
195 // `use_visible_rect` is set to true, only `VideoFrame::visible_rect()`
196 // portion is copied, otherwise copies all underlying buffer.
197 [[nodiscard]] gpu::SyncToken CopyVideoFrameToSharedImage(
198 viz::RasterContextProvider* raster_context_provider,
199 scoped_refptr<VideoFrame> video_frame,
200 const gpu::MailboxHolder& destination,
201 bool use_visible_rect);
203 // In general, We hold the most recently painted frame to increase the
204 // performance for the case that the same frame needs to be painted
205 // repeatedly. Call this function if you are sure the most recent frame will
206 // never be painted again, so we can release the resource.
209 // Used for unit test.
210 gfx::Size LastImageDimensionsForTesting();
213 // This structure wraps information extracted out of a VideoFrame and/or
214 // constructed out of it. The various calls in PaintCanvasVideoRenderer must
215 // not keep a reference to the VideoFrame so necessary data is extracted out
218 explicit Cache(VideoFrame::ID frame_id);
221 // VideoFrame::unique_id() of the videoframe used to generate the cache.
222 VideoFrame::ID frame_id;
224 // A PaintImage that can be used to draw into a PaintCanvas. This is sized
225 // to the visible size of the VideoFrame. Its contents are generated lazily.
226 cc::PaintImage paint_image;
228 // The backing for the source texture. This is also responsible for managing
229 // the lifetime of the texture.
230 sk_sp<VideoTextureBacking> texture_backing;
232 // The GL texture ID used in non-OOP code path.
233 // This is only set if the VideoFrame was texture-backed.
234 uint32_t source_texture = 0;
236 // The allocated size of VideoFrame texture.
237 // This is only set if the VideoFrame was texture-backed.
238 gfx::Size coded_size;
240 // The visible subrect of |coded_size| that represents the logical contents
241 // of the frame after cropping.
242 // This is only set if the VideoFrame was texture-backed.
243 gfx::Rect visible_rect;
245 // True if the underlying resource was created with a top left origin.
246 bool texture_origin_is_top_left = true;
248 // Used to allow recycling of the previous shared image. This requires that
249 // no external users have access to this resource via SkImage. Returns true
250 // if the existing resource can be recycled.
254 // Update the cache holding the most-recently-painted frame. Returns false
255 // if the image couldn't be updated.
256 bool UpdateLastImage(scoped_refptr<VideoFrame> video_frame,
257 viz::RasterContextProvider* raster_context_provider,
258 bool allow_wrap_texture);
260 bool PrepareVideoFrame(scoped_refptr<VideoFrame> video_frame,
261 viz::RasterContextProvider* raster_context_provider,
262 const gpu::MailboxHolder& dest_holder);
264 bool UploadVideoFrameToGLTexture(
265 viz::RasterContextProvider* raster_context_provider,
266 gpu::gles2::GLES2Interface* destination_gl,
267 const gpu::Capabilities& destination_gl_capabilities,
268 scoped_refptr<VideoFrame> video_frame,
270 unsigned int texture,
271 unsigned int internal_format,
276 bool CacheBackingWrapsTexture() const;
278 absl::optional<Cache> cache_;
280 // If |cache_| is not used for a while, it's deleted to save memory.
281 base::DelayTimer cache_deleting_timer_;
282 // Stable paint image id to provide to draw image calls.
283 cc::PaintImage::Id renderer_stable_id_;
285 // Used for DCHECKs to ensure method calls executed in the correct thread.
286 base::SequenceChecker sequence_checker_;
288 struct YUVTextureCache {
293 // The ContextProvider that holds the texture.
294 scoped_refptr<viz::RasterContextProvider> raster_context_provider;
296 // The size of the texture.
299 // The shared image backing the texture.
300 gpu::Mailbox mailbox;
302 // Used to perform YUV->RGB conversion on video frames. Internally caches
303 // shared images that are created to upload CPU video frame data to the GPU.
304 VideoFrameYUVConverter yuv_converter;
306 // A SyncToken after last usage, used for reusing or destroying texture and
308 gpu::SyncToken sync_token;
310 YUVTextureCache yuv_cache_;
315 #endif // MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_