1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_
6 #define MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_
14 #include "base/memory/raw_ptr.h"
15 #include "base/memory/scoped_refptr.h"
16 #include "base/memory/weak_ptr.h"
17 #include "base/trace_event/memory_dump_provider.h"
18 #include "base/unguessable_token.h"
19 #include "components/viz/common/resources/release_callback.h"
20 #include "components/viz/common/resources/resource_id.h"
21 #include "components/viz/common/resources/transferable_resource.h"
22 #include "gpu/command_buffer/client/gles2_interface.h"
23 #include "gpu/command_buffer/client/raster_interface.h"
24 #include "media/base/media_export.h"
25 #include "media/base/video_frame.h"
26 #include "ui/gfx/buffer_types.h"
27 #include "ui/gfx/geometry/size.h"
35 class ClientResourceProvider;
36 class RasterContextProvider;
37 class CompositorRenderPass;
38 class SharedBitmapReporter;
46 class PaintCanvasVideoRenderer;
48 // Specifies what type of data is contained in the mailboxes, as well as how
49 // many mailboxes will be present.
50 enum class VideoFrameResourceType {
57 #if defined(TIZEN_VIDEO_HOLE)
58 // TODO(danakj): Implement this with a solid color layer instead of a video
59 // frame and video layer.
63 // The VideoFrame is merely a hint to compositor that a hole must be made
64 // transparent so the video underlay will be visible.
65 // Used by Chromecast only.
69 class MEDIA_EXPORT VideoFrameExternalResources {
71 VideoFrameResourceType type = VideoFrameResourceType::NONE;
72 std::vector<viz::TransferableResource> resources;
73 std::vector<viz::ReleaseCallback> release_callbacks;
75 // Used by hardware textures which do not return values in the 0-1 range.
76 // After a lookup, subtract offset and multiply by multiplier.
78 float multiplier = 1.f;
79 uint32_t bits_per_channel = 8;
81 VideoFrameExternalResources();
82 VideoFrameExternalResources(VideoFrameExternalResources&& other);
83 VideoFrameExternalResources& operator=(VideoFrameExternalResources&& other);
84 ~VideoFrameExternalResources();
87 // VideoResourceUpdater is used by the video system to produce frame content as
88 // resources consumable by the display compositor.
89 class MEDIA_EXPORT VideoResourceUpdater
90 : public base::trace_event::MemoryDumpProvider {
92 // For GPU compositing |context_provider| should be provided and for software
93 // compositing |shared_bitmap_reporter| should be provided. If there is a
94 // non-null |context_provider| we assume GPU compositing.
95 VideoResourceUpdater(viz::RasterContextProvider* context_provider,
96 viz::SharedBitmapReporter* shared_bitmap_reporter,
97 viz::ClientResourceProvider* resource_provider,
98 bool use_stream_video_draw_quad,
99 bool use_gpu_memory_buffer_resources,
100 int max_resource_size);
102 VideoResourceUpdater(const VideoResourceUpdater&) = delete;
103 VideoResourceUpdater& operator=(const VideoResourceUpdater&) = delete;
105 ~VideoResourceUpdater() override;
107 // For each CompositorFrame the following sequence is expected:
108 // 1. ObtainFrameResources(): Import resources for the next video frame with
109 // viz::ClientResourceProvider. This will reuse existing GPU or
110 // SharedMemory buffers if possible, otherwise it will allocate new ones.
111 // 2. AppendQuads(): Add DrawQuads to CompositorFrame for video.
112 // 3. ReleaseFrameResources(): After the CompositorFrame has been submitted,
113 // remove imported resources from viz::ClientResourceProvider.
114 void ObtainFrameResources(
115 scoped_refptr<VideoFrame> video_frame,
116 viz::ResourceId resource_id = viz::kInvalidResourceId);
117 void ReleaseFrameResources();
118 // Appends a quad representing |frame| to |render_pass|.
119 // At most one quad is expected to be appended, this is enforced by the users
120 // of this class (e.g: VideoFrameSubmitter). Producing only one quad will
121 // allow viz to optimize compositing when the only content changing per-frame
123 void AppendQuads(viz::CompositorRenderPass* render_pass,
124 scoped_refptr<VideoFrame> frame,
125 gfx::Transform transform,
127 gfx::Rect visible_quad_rect,
128 const gfx::MaskFilterInfo& mask_filter_info,
129 absl::optional<gfx::Rect> clip_rect,
132 int sorting_context_id);
134 // TODO(kylechar): This is only public for testing, make private.
135 VideoFrameExternalResources CreateExternalResourcesFromVideoFrame(
136 scoped_refptr<VideoFrame> video_frame);
138 viz::SharedImageFormat YuvSharedImageFormat(int bits_per_channel);
142 class HardwarePlaneResource;
143 class SoftwarePlaneResource;
145 // A resource that will be embedded in a DrawQuad in the next CompositorFrame.
146 // Each video plane will correspond to one FrameResource.
147 struct FrameResource {
149 FrameResource(viz::ResourceId id, const gfx::Size& size);
152 gfx::Size size_in_pixels;
155 bool software_compositor() const { return context_provider_ == nullptr; }
157 // Reallocate |upload_pixels_| with the requested size.
158 bool ReallocateUploadPixels(size_t needed_size);
160 // Obtain a resource of the right format by either recycling an
161 // unreferenced but appropriately formatted resource, or by
162 // allocating a new resource.
163 // Additionally, if the |unique_id| and |plane_index| match, then
164 // it is assumed that the resource has the right data already and will only be
165 // used for reading, and so is returned even if it is still referenced.
166 // Passing -1 for |plane_index| avoids returning referenced
168 PlaneResource* RecycleOrAllocateResource(const gfx::Size& resource_size,
169 viz::SharedImageFormat si_format,
170 const gfx::ColorSpace& color_space,
171 VideoFrame::ID unique_id,
173 PlaneResource* AllocateResource(const gfx::Size& plane_size,
174 viz::SharedImageFormat format,
175 const gfx::ColorSpace& color_space);
177 // Create a copy of a texture-backed source video frame in a new GL_TEXTURE_2D
178 // texture. This is used when there are multiple GPU threads (Android WebView)
179 // and the source video frame texture can't be used on the output GL context.
180 // https://crbug.com/582170
181 void CopyHardwarePlane(VideoFrame* video_frame,
182 const gfx::ColorSpace& resource_color_space,
183 const gpu::MailboxHolder& mailbox_holder,
184 VideoFrameExternalResources* external_resources);
186 // Get resources ready to be appended into DrawQuads. This is used for GPU
187 // compositing most of the time, except for the cases mentioned in
188 // CreateForSoftwarePlanes().
189 VideoFrameExternalResources CreateForHardwarePlanes(
190 scoped_refptr<VideoFrame> video_frame);
192 // Get resources ready to be appended into DrawQuads. This is always used for
193 // software compositing. This is also used for GPU compositing when the input
194 // video frame has no textures.
195 VideoFrameExternalResources CreateForSoftwarePlanes(
196 scoped_refptr<VideoFrame> video_frame);
198 gpu::gles2::GLES2Interface* ContextGL();
199 gpu::raster::RasterInterface* RasterInterface();
200 gpu::InterfaceBase* InterfaceBase();
202 void RecycleResource(uint32_t plane_resource_id,
203 const gpu::SyncToken& sync_token,
205 void ReturnTexture(scoped_refptr<VideoFrame> video_frame,
206 const gpu::SyncToken& original_release_token,
207 const gpu::SyncToken& new_release_token,
210 // base::trace_event::MemoryDumpProvider implementation.
211 bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
212 base::trace_event::ProcessMemoryDump* pmd) override;
214 const raw_ptr<viz::RasterContextProvider> context_provider_;
215 const raw_ptr<viz::SharedBitmapReporter> shared_bitmap_reporter_;
216 const raw_ptr<viz::ClientResourceProvider, DanglingUntriaged>
218 const bool use_stream_video_draw_quad_;
219 const bool use_gpu_memory_buffer_resources_;
220 const int max_resource_size_;
221 const int tracing_id_;
222 std::unique_ptr<PaintCanvasVideoRenderer> video_renderer_;
223 uint32_t next_plane_resource_id_ = 1;
225 // Temporary pixel buffer when converting between formats.
226 std::unique_ptr<uint8_t[], base::UncheckedFreeDeleter> upload_pixels_;
227 size_t upload_pixels_size_ = 0;
229 VideoFrameResourceType frame_resource_type_;
231 float frame_resource_offset_;
232 float frame_resource_multiplier_;
233 uint32_t frame_bits_per_channel_;
235 // Resources that will be placed into quads by the next call to
236 // AppendDrawQuads().
237 std::vector<FrameResource> frame_resources_;
238 // If the video resource is a hole punching VideoFrame sent by Chromecast,
239 // the VideoFrame carries an |overlay_plane_id_| to activate the video
240 // overlay, but there is no video content to display within VideoFrame.
241 base::UnguessableToken overlay_plane_id_;
243 // Resources allocated by VideoResourceUpdater. Used to recycle resources so
244 // we can reduce the number of allocations and data transfers.
245 std::vector<std::unique_ptr<PlaneResource>> all_resources_;
247 base::WeakPtrFactory<VideoResourceUpdater> weak_ptr_factory_{this};
252 #endif // MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_