1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // VideoCaptureDevice is the abstract base class for realizing video capture
6 // device support in Chromium. It provides the interface for OS dependent
8 // The class is created and functions are invoked on a thread owned by
9 // VideoCaptureManager. Capturing is done on other threads, depending on the OS
10 // specific implementation.
12 #ifndef MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
13 #define MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
22 #include "base/callback.h"
23 #include "base/files/file.h"
24 #include "base/memory/ref_counted.h"
25 #include "base/memory/unsafe_shared_memory_region.h"
26 #include "base/single_thread_task_runner.h"
27 #include "base/time/time.h"
28 #include "build/build_config.h"
29 #include "media/base/video_frame.h"
30 #include "media/capture/capture_export.h"
31 #include "media/capture/mojom/image_capture.mojom.h"
32 #include "media/capture/video/video_capture_buffer_handle.h"
33 #include "media/capture/video/video_capture_device_descriptor.h"
34 #include "media/capture/video/video_capture_feedback.h"
35 #include "media/capture/video_capture_types.h"
36 #include "ui/gfx/gpu_memory_buffer.h"
44 class CAPTURE_EXPORT VideoFrameConsumerFeedbackObserver {
46 virtual ~VideoFrameConsumerFeedbackObserver() {}
48 // During processing of a video frame, consumers may report back their
49 // utilization level to the source device. The device may use this information
50 // to adjust the rate of data it pushes out. Values are interpreted as
52 // Less than 0.0 is meaningless and should be ignored. 1.0 indicates a
53 // maximum sustainable utilization. Greater than 1.0 indicates the consumer
54 // is likely to stall or drop frames if the data volume is not reduced.
56 // Example: In a system that encodes and transmits video frames over the
57 // network, this value can be used to indicate whether sufficient CPU
58 // is available for encoding and/or sufficient bandwidth is available for
59 // transmission over the network. The maximum of the two utilization
60 // measurements would be used as feedback.
62 // The parameter |frame_feedback_id| must match a |frame_feedback_id|
63 // previously sent out by the VideoCaptureDevice we are giving feedback about.
64 // It is used to indicate which particular frame the reported utilization
66 virtual void OnUtilizationReport(int frame_feedback_id,
67 media::VideoCaptureFeedback feedback) {}
70 struct CAPTURE_EXPORT CapturedExternalVideoBuffer {
71 CapturedExternalVideoBuffer(gfx::GpuMemoryBufferHandle handle,
72 VideoCaptureFormat format,
73 gfx::ColorSpace color_space);
74 CapturedExternalVideoBuffer(CapturedExternalVideoBuffer&& other);
75 ~CapturedExternalVideoBuffer();
77 CapturedExternalVideoBuffer& operator=(CapturedExternalVideoBuffer&& other);
79 gfx::GpuMemoryBufferHandle handle;
80 VideoCaptureFormat format;
81 gfx::ColorSpace color_space;
84 class CAPTURE_EXPORT VideoCaptureDevice
85 : public VideoFrameConsumerFeedbackObserver {
88 // Interface defining the methods that clients of VideoCapture must have. It
89 // is actually two-in-one: clients may implement OnIncomingCapturedData() or
90 // ReserveOutputBuffer() + OnIncomingCapturedVideoFrame(), or all of them.
91 // All methods may be called as soon as AllocateAndStart() of the
92 // corresponding VideoCaptureDevice is invoked. The methods for buffer
93 // reservation and frame delivery may be called from arbitrary threads but
94 // are guaranteed to be called non-concurrently. The status reporting methods
95 // (OnStarted, OnLog, OnError) may be called concurrently.
96 class CAPTURE_EXPORT Client {
98 // Struct bundling several parameters being passed between a
99 // VideoCaptureDevice and its VideoCaptureDevice::Client.
100 struct CAPTURE_EXPORT Buffer {
102 // Destructor-only interface for encapsulating scoped access permission to
104 class CAPTURE_EXPORT ScopedAccessPermission {
106 virtual ~ScopedAccessPermission() {}
109 class CAPTURE_EXPORT HandleProvider {
111 virtual ~HandleProvider() {}
113 // Duplicate as an writable (unsafe) shared memory region.
114 virtual base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() = 0;
116 // Duplicate as a writable (unsafe) mojo buffer.
117 virtual mojo::ScopedSharedBufferHandle DuplicateAsMojoBuffer() = 0;
119 // Access a |VideoCaptureBufferHandle| for local, writable memory.
120 virtual std::unique_ptr<VideoCaptureBufferHandle>
121 GetHandleForInProcessAccess() = 0;
123 // Clone a |GpuMemoryBufferHandle| for IPC.
124 virtual gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() = 0;
128 Buffer(int buffer_id,
129 int frame_feedback_id,
130 std::unique_ptr<HandleProvider> handle_provider,
131 std::unique_ptr<ScopedAccessPermission> access_permission);
133 Buffer(Buffer&& other);
134 Buffer& operator=(Buffer&& other);
137 int frame_feedback_id;
138 std::unique_ptr<HandleProvider> handle_provider;
139 std::unique_ptr<ScopedAccessPermission> access_permission;
141 // Some buffer types may be preemptively mapped in the capturer if
142 // requested by the consumer.
143 // This is used to notify the client that a shared memory region
144 // associated with the buffer is valid.
145 bool is_premapped = false;
148 // Result code for calls to ReserveOutputBuffer()
149 enum class ReserveResult {
151 kMaxBufferCountExceeded,
157 // Captured a new video frame, data for which is pointed to by |data|.
159 // The format of the frame is described by |frame_format|, and is assumed to
160 // be tightly packed. This method will try to reserve an output buffer and
161 // copy from |data| into the output buffer. If no output buffer is
162 // available, the frame will be silently dropped. |reference_time| is
163 // system clock time when we detect the capture happens, it is used for
164 // Audio/Video sync, not an exact presentation time for playout, because it
165 // could contain noise. |timestamp| measures the ideal time span between the
166 // first frame in the stream and the current frame; however, the time source
167 // is determined by the platform's device driver and is often not the system
168 // clock, or even has a drift with respect to system clock.
169 // |frame_feedback_id| is an identifier that allows clients to refer back to
170 // this particular frame when reporting consumer feedback via
171 // OnConsumerReportingUtilization(). This identifier is needed because
172 // frames are consumed asynchronously and multiple frames can be "in flight"
174 // TODO(crbug.com/978143): remove |frame_feedback_id| default value.
175 virtual void OnIncomingCapturedData(const uint8_t* data,
177 const VideoCaptureFormat& frame_format,
178 const gfx::ColorSpace& color_space,
179 int clockwise_rotation,
181 base::TimeTicks reference_time,
182 base::TimeDelta timestamp,
183 int frame_feedback_id = 0) = 0;
185 // Captured a new video frame, data for which is stored in the
186 // GpuMemoryBuffer pointed to by |buffer|. The format of the frame is
187 // described by |frame_format|. Since the memory buffer pointed to by
188 // |buffer| may be allocated with some size/address alignment requirement,
189 // this method takes into consideration the size and offset of each plane in
190 // |buffer| when creating the content of the output buffer.
191 // |clockwise_rotation|, |reference_time|, |timestamp|, and
192 // |frame_feedback_id| serve the same purposes as in OnIncomingCapturedData.
193 // TODO(crbug.com/978143): remove |frame_feedback_id| default value.
194 virtual void OnIncomingCapturedGfxBuffer(
195 gfx::GpuMemoryBuffer* buffer,
196 const VideoCaptureFormat& frame_format,
197 int clockwise_rotation,
198 base::TimeTicks reference_time,
199 base::TimeDelta timestamp,
200 int frame_feedback_id = 0) = 0;
202 // Captured a new video frame. The data for this frame is in |handle|,
203 // which is owned by the platform-specific capture device. It is the
204 // responsibilty of the implementation to prevent the buffer in |handle|
205 // from being reused by the external capturer. In practice, this is used
206 // only on macOS, the external capturer maintains a CVPixelBufferPool, and
207 // gfx::ScopedInUseIOSurface is used to prevent reuse of buffers until all
208 // consumers have consumed them.
209 virtual void OnIncomingCapturedExternalBuffer(
210 CapturedExternalVideoBuffer buffer,
211 std::vector<CapturedExternalVideoBuffer> scaled_buffers,
212 base::TimeTicks reference_time,
213 base::TimeDelta timestamp) = 0;
215 // Reserve an output buffer into which contents can be captured directly.
216 // The returned |buffer| will always be allocated with a memory size
217 // suitable for holding a packed video frame with pixels of |format| format,
218 // of |dimensions| frame dimensions. It is permissible for |dimensions| to
219 // be zero; in which case the returned Buffer does not guarantee memory
220 // backing, but functions as a reservation for external input for the
221 // purposes of buffer throttling.
223 // The buffer stays reserved for use by the caller as long as it
224 // holds on to the contained |buffer_read_write_permission|.
225 virtual ReserveResult ReserveOutputBuffer(const gfx::Size& dimensions,
226 VideoPixelFormat format,
227 int frame_feedback_id,
229 WARN_UNUSED_RESULT = 0;
231 // Provides VCD::Client with a populated Buffer containing the content of
232 // the next video frame. The |buffer| must originate from an earlier call to
233 // ReserveOutputBuffer().
234 // See OnIncomingCapturedData for details of |reference_time| and
236 virtual void OnIncomingCapturedBuffer(Buffer buffer,
237 const VideoCaptureFormat& format,
238 base::TimeTicks reference_time,
239 base::TimeDelta timestamp) = 0;
241 // Extended version of OnIncomingCapturedBuffer() allowing clients to
242 // pass a custom |visible_rect| and |additional_metadata|.
243 virtual void OnIncomingCapturedBufferExt(
245 const VideoCaptureFormat& format,
246 const gfx::ColorSpace& color_space,
247 base::TimeTicks reference_time,
248 base::TimeDelta timestamp,
249 gfx::Rect visible_rect,
250 const VideoFrameMetadata& additional_metadata) = 0;
252 // An error has occurred that cannot be handled and VideoCaptureDevice must
253 // be StopAndDeAllocate()-ed. |reason| is a text description of the error.
254 virtual void OnError(VideoCaptureError error,
255 const base::Location& from_here,
256 const std::string& reason) = 0;
258 virtual void OnFrameDropped(VideoCaptureFrameDropReason reason) = 0;
260 // VideoCaptureDevice requests the |message| to be logged.
261 virtual void OnLog(const std::string& message) {}
263 // Returns the current buffer pool utilization, in the range 0.0 (no buffers
264 // are in use by producers or consumers) to 1.0 (all buffers are in use).
265 virtual double GetBufferPoolUtilization() const = 0;
267 // VideoCaptureDevice reports it's successfully started.
268 virtual void OnStarted() = 0;
271 ~VideoCaptureDevice() override;
273 // Prepares the video capturer for use. StopAndDeAllocate() must be called
274 // before the object is deleted.
275 virtual void AllocateAndStart(const VideoCaptureParams& params,
276 std::unique_ptr<Client> client) = 0;
278 // In cases where the video capturer self-pauses (e.g., a screen capturer
279 // where the screen's content has not changed in a while), consumers may call
280 // this to request a "refresh frame" be delivered to the Client. This is used
281 // in a number of circumstances, such as:
283 // 1. An additional consumer of video frames is starting up and requires a
284 // first frame (as opposed to not receiving a frame for an indeterminate
286 // 2. A few repeats of the same frame would allow a lossy video encoder to
287 // improve the video quality of unchanging content.
289 // The default implementation is a no-op. VideoCaptureDevice implementations
290 // are not required to honor this request, especially if they do not
291 // self-pause and/or if honoring the request would cause them to exceed their
292 // configured maximum frame rate. Any VideoCaptureDevice that does self-pause,
293 // however, should provide an implementation of this method that makes
294 // reasonable attempts to honor these requests.
296 // Note: This should only be called after AllocateAndStart() and before
297 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
298 virtual void RequestRefreshFrame() {}
300 // Optionally suspends frame delivery. The VideoCaptureDevice may or may not
301 // honor this request. Thus, the caller cannot assume frame delivery will
302 // actually stop. Even if frame delivery is suspended, this might not take
303 // effect immediately.
305 // The purpose of this is to quickly place the device into a state where it's
306 // resource utilization is minimized while there are no frame consumers; and
307 // then quickly resume once a frame consumer is present.
309 // Note: This should only be called after AllocateAndStart() and before
310 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
311 virtual void MaybeSuspend() {}
313 // Resumes frame delivery, if it was suspended. If frame delivery was not
314 // suspended, this is a no-op, and frame delivery will continue.
316 // Note: This should only be called after AllocateAndStart() and before
317 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
318 virtual void Resume() {}
320 // Deallocates the video capturer, possibly asynchronously.
322 // This call requires the device to do the following things, eventually: put
323 // hardware into a state where other applications could use it, free the
324 // memory associated with capture, and delete the |client| pointer passed into
327 // If deallocation is done asynchronously, then the device implementation must
328 // ensure that a subsequent AllocateAndStart() operation targeting the same ID
329 // would be sequenced through the same task runner, so that deallocation
331 virtual void StopAndDeAllocate() = 0;
333 // Hints to the source that if it has an alpha channel, that alpha channel
334 // will be ignored and can be discarded.
335 virtual void SetCanDiscardAlpha(bool can_discard_alpha) {}
337 // Retrieve the photo capabilities and settings of the device (e.g. zoom
338 // levels etc). On success, invokes |callback|. On failure, drops callback
339 // without invoking it.
340 using GetPhotoStateCallback = base::OnceCallback<void(mojom::PhotoStatePtr)>;
341 virtual void GetPhotoState(GetPhotoStateCallback callback);
343 // On success, invokes |callback| with value |true|. On failure, drops
344 // callback without invoking it.
345 using SetPhotoOptionsCallback = base::OnceCallback<void(bool)>;
346 virtual void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
347 SetPhotoOptionsCallback callback);
349 // Asynchronously takes a photo, possibly reconfiguring the capture objects
350 // and/or interrupting the capture flow. Runs |callback|, if the photo was
351 // successfully taken. On failure, drops callback without invoking it.
352 // Note that |callback| may be runned on a thread different than the thread
353 // where TakePhoto() was called.
354 using TakePhotoCallback = base::OnceCallback<void(mojom::BlobPtr blob)>;
355 virtual void TakePhoto(TakePhotoCallback callback);
357 // Gets the power line frequency, either from the params if specified by the
358 // user or from the current system time zone.
359 static PowerLineFrequency GetPowerLineFrequency(
360 const VideoCaptureParams& params);
363 // Gets the power line frequency from the current system time zone if this is
364 // defined, otherwise returns 0.
365 static PowerLineFrequency GetPowerLineFrequencyForLocation();
368 VideoCaptureFrameDropReason ConvertReservationFailureToFrameDropReason(
369 VideoCaptureDevice::Client::ReserveResult reserve_result);
373 #endif // MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_