1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // VideoCaptureDevice is the abstract base class for realizing video capture
6 // device support in Chromium. It provides the interface for OS dependent
8 // The class is created and functions are invoked on a thread owned by
9 // VideoCaptureManager. Capturing is done on other threads, depending on the OS
10 // specific implementation.
12 #ifndef MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
13 #define MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
21 #include "base/files/file.h"
22 #include "base/functional/callback.h"
23 #include "base/memory/unsafe_shared_memory_region.h"
24 #include "base/task/single_thread_task_runner.h"
25 #include "base/time/time.h"
26 #include "base/token.h"
27 #include "build/build_config.h"
28 #include "media/base/video_frame.h"
29 #include "media/capture/capture_export.h"
30 #include "media/capture/mojom/image_capture.mojom.h"
31 #include "media/capture/mojom/video_capture_types.mojom.h"
32 #include "media/capture/video/video_capture_buffer_handle.h"
33 #include "media/capture/video/video_capture_device_descriptor.h"
34 #include "media/capture/video/video_capture_feedback.h"
35 #include "media/capture/video_capture_types.h"
36 #include "ui/gfx/gpu_memory_buffer.h"
39 #include <mfobjects.h>
40 #include <wrl/client.h>
49 class CAPTURE_EXPORT VideoFrameConsumerFeedbackObserver {
51 virtual ~VideoFrameConsumerFeedbackObserver() {}
53 // During processing of a video frame, consumers may report back their
54 // utilization level to the source device. The device may use this information
55 // to adjust the rate of data it pushes out. Values are interpreted as
57 // Less than 0.0 is meaningless and should be ignored. 1.0 indicates a
58 // maximum sustainable utilization. Greater than 1.0 indicates the consumer
59 // is likely to stall or drop frames if the data volume is not reduced.
61 // Example: In a system that encodes and transmits video frames over the
62 // network, this value can be used to indicate whether sufficient CPU
63 // is available for encoding and/or sufficient bandwidth is available for
64 // transmission over the network. The maximum of the two utilization
65 // measurements would be used as feedback.
66 virtual void OnUtilizationReport(media::VideoCaptureFeedback feedback) {}
69 struct CAPTURE_EXPORT CapturedExternalVideoBuffer {
70 CapturedExternalVideoBuffer(gfx::GpuMemoryBufferHandle handle,
71 VideoCaptureFormat format,
72 gfx::ColorSpace color_space);
75 CapturedExternalVideoBuffer(Microsoft::WRL::ComPtr<IMFMediaBuffer> imf_buffer,
76 gfx::GpuMemoryBufferHandle handle,
77 VideoCaptureFormat format,
78 gfx::ColorSpace color_space);
81 CapturedExternalVideoBuffer(CapturedExternalVideoBuffer&& other);
82 CapturedExternalVideoBuffer& operator=(CapturedExternalVideoBuffer&& other);
84 CapturedExternalVideoBuffer(const CapturedExternalVideoBuffer&) = delete;
85 CapturedExternalVideoBuffer& operator=(const CapturedExternalVideoBuffer&) =
88 ~CapturedExternalVideoBuffer();
91 Microsoft::WRL::ComPtr<IMFMediaBuffer> imf_buffer;
93 gfx::GpuMemoryBufferHandle handle;
94 VideoCaptureFormat format;
95 gfx::ColorSpace color_space;
98 class CAPTURE_EXPORT VideoCaptureDevice
99 : public VideoFrameConsumerFeedbackObserver {
102 // Interface defining the methods that clients of VideoCapture must have. It
103 // is actually two-in-one: clients may implement OnIncomingCapturedData() or
104 // ReserveOutputBuffer() + OnIncomingCapturedVideoFrame(), or all of them.
105 // All methods may be called as soon as AllocateAndStart() of the
106 // corresponding VideoCaptureDevice is invoked. The methods for buffer
107 // reservation and frame delivery may be called from arbitrary threads but
108 // are guaranteed to be called non-concurrently. The status reporting methods
109 // (OnStarted, OnLog, OnError) may be called concurrently.
110 class CAPTURE_EXPORT Client {
112 // Struct bundling several parameters being passed between a
113 // VideoCaptureDevice and its VideoCaptureDevice::Client.
114 struct CAPTURE_EXPORT Buffer {
116 // Destructor-only interface for encapsulating scoped access permission to
118 class CAPTURE_EXPORT ScopedAccessPermission {
120 virtual ~ScopedAccessPermission() {}
123 class CAPTURE_EXPORT HandleProvider {
125 virtual ~HandleProvider() {}
127 // Duplicate as an writable (unsafe) shared memory region.
128 virtual base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() = 0;
130 // Access a |VideoCaptureBufferHandle| for local, writable memory.
131 virtual std::unique_ptr<VideoCaptureBufferHandle>
132 GetHandleForInProcessAccess() = 0;
134 // Clone a |GpuMemoryBufferHandle| for IPC.
135 virtual gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() = 0;
139 Buffer(int buffer_id,
140 int frame_feedback_id,
141 std::unique_ptr<HandleProvider> handle_provider,
142 std::unique_ptr<ScopedAccessPermission> access_permission);
144 Buffer(Buffer&& other);
145 Buffer& operator=(Buffer&& other);
148 int frame_feedback_id;
149 std::unique_ptr<HandleProvider> handle_provider;
150 std::unique_ptr<ScopedAccessPermission> access_permission;
152 // Some buffer types may be preemptively mapped in the capturer if
153 // requested by the consumer.
154 // This is used to notify the client that a shared memory region
155 // associated with the buffer is valid.
156 bool is_premapped = false;
159 // Result code for calls to ReserveOutputBuffer()
160 enum class ReserveResult {
162 kMaxBufferCountExceeded,
168 // The configuration of the VideoCaptureDevice has changed.
169 virtual void OnCaptureConfigurationChanged() = 0;
171 // Captured a new video frame, data for which is pointed to by |data|.
173 // The format of the frame is described by |frame_format|, and is assumed to
174 // be tightly packed. This method will try to reserve an output buffer and
175 // copy from |data| into the output buffer. If no output buffer is
176 // available, the frame will be silently dropped. |reference_time| is
177 // system clock time when we detect the capture happens, it is used for
178 // Audio/Video sync, not an exact presentation time for playout, because it
179 // could contain noise. |timestamp| measures the ideal time span between the
180 // first frame in the stream and the current frame; however, the time source
181 // is determined by the platform's device driver and is often not the system
182 // clock, or even has a drift with respect to system clock.
183 // |frame_feedback_id| is an identifier that allows clients to refer back to
184 // this particular frame when reporting consumer feedback via
185 // OnConsumerReportingUtilization(). This identifier is needed because
186 // frames are consumed asynchronously and multiple frames can be "in flight"
188 virtual void OnIncomingCapturedData(const uint8_t* data,
190 const VideoCaptureFormat& frame_format,
191 const gfx::ColorSpace& color_space,
192 int clockwise_rotation,
194 base::TimeTicks reference_time,
195 base::TimeDelta timestamp,
196 int frame_feedback_id) = 0;
197 // Convenience wrapper that passes in 0 as |frame_feedback_id|.
198 void OnIncomingCapturedData(const uint8_t* data,
200 const VideoCaptureFormat& frame_format,
201 const gfx::ColorSpace& color_space,
202 int clockwise_rotation,
204 base::TimeTicks reference_time,
205 base::TimeDelta timestamp);
207 // Captured a new video frame, data for which is stored in the
208 // GpuMemoryBuffer pointed to by |buffer|. The format of the frame is
209 // described by |frame_format|. Since the memory buffer pointed to by
210 // |buffer| may be allocated with some size/address alignment requirement,
211 // this method takes into consideration the size and offset of each plane in
212 // |buffer| when creating the content of the output buffer.
213 // |clockwise_rotation|, |reference_time|, |timestamp|, and
214 // |frame_feedback_id| serve the same purposes as in OnIncomingCapturedData.
215 virtual void OnIncomingCapturedGfxBuffer(
216 gfx::GpuMemoryBuffer* buffer,
217 const VideoCaptureFormat& frame_format,
218 int clockwise_rotation,
219 base::TimeTicks reference_time,
220 base::TimeDelta timestamp,
221 int frame_feedback_id) = 0;
222 // Convenience wrapper that passes in 0 as |frame_feedback_id|.
223 void OnIncomingCapturedGfxBuffer(gfx::GpuMemoryBuffer* buffer,
224 const VideoCaptureFormat& frame_format,
225 int clockwise_rotation,
226 base::TimeTicks reference_time,
227 base::TimeDelta timestamp);
229 // Captured a new video frame. The data for this frame is in
230 // |buffer.handle|, which is owned by the platform-specific capture device.
231 // It is the responsibility of the implementation to prevent the buffer in
232 // |buffer.handle| from being reused by the external capturer. In practice,
233 // this is used only on macOS, the external capturer maintains a
234 // CVPixelBufferPool, and gfx::ScopedInUseIOSurface is used to prevent reuse
235 // of buffers until all consumers have consumed them. |visible_rect|
236 // specifies the region in the memory pointed to by |buffer.handle| that
237 // contains the captured content.
238 virtual void OnIncomingCapturedExternalBuffer(
239 CapturedExternalVideoBuffer buffer,
240 base::TimeTicks reference_time,
241 base::TimeDelta timestamp,
242 const gfx::Rect& visible_rect) = 0;
244 // Reserve an output buffer into which contents can be captured directly.
245 // The returned |buffer| will always be allocated with a memory size
246 // suitable for holding a packed video frame with pixels of |format| format,
247 // of |dimensions| frame dimensions. It is permissible for |dimensions| to
248 // be zero; in which case the returned Buffer does not guarantee memory
249 // backing, but functions as a reservation for external input for the
250 // purposes of buffer throttling.
252 // The buffer stays reserved for use by the caller as long as it
253 // holds on to the contained |buffer_read_write_permission|.
254 [[nodiscard]] virtual ReserveResult ReserveOutputBuffer(
255 const gfx::Size& dimensions,
256 VideoPixelFormat format,
257 int frame_feedback_id,
260 // Provides VCD::Client with a populated Buffer containing the content of
261 // the next video frame. The |buffer| must originate from an earlier call to
262 // ReserveOutputBuffer().
263 // See OnIncomingCapturedData for details of |reference_time| and
265 virtual void OnIncomingCapturedBuffer(Buffer buffer,
266 const VideoCaptureFormat& format,
267 base::TimeTicks reference_time,
268 base::TimeDelta timestamp) = 0;
270 // Extended version of OnIncomingCapturedBuffer() allowing clients to
271 // pass a custom |visible_rect| and |additional_metadata|.
272 virtual void OnIncomingCapturedBufferExt(
274 const VideoCaptureFormat& format,
275 const gfx::ColorSpace& color_space,
276 base::TimeTicks reference_time,
277 base::TimeDelta timestamp,
278 gfx::Rect visible_rect,
279 const VideoFrameMetadata& additional_metadata,
280 unsigned int encoded_data_size = 0) = 0;
282 // An error has occurred that cannot be handled and VideoCaptureDevice must
283 // be StopAndDeAllocate()-ed. |reason| is a text description of the error.
284 virtual void OnError(VideoCaptureError error,
285 const base::Location& from_here,
286 const std::string& reason) = 0;
288 virtual void OnFrameDropped(VideoCaptureFrameDropReason reason) = 0;
290 // VideoCaptureDevice requests the |message| to be logged.
291 virtual void OnLog(const std::string& message) {}
293 // Returns the current buffer pool utilization, in the range 0.0 (no buffers
294 // are in use by producers or consumers) to 1.0 (all buffers are in use).
295 virtual double GetBufferPoolUtilization() const = 0;
297 // VideoCaptureDevice reports it's successfully started.
298 virtual void OnStarted() = 0;
301 ~VideoCaptureDevice() override;
303 // Prepares the video capturer for use. StopAndDeAllocate() must be called
304 // before the object is deleted.
305 virtual void AllocateAndStart(const VideoCaptureParams& params,
306 std::unique_ptr<Client> client) = 0;
308 // In cases where the video capturer self-pauses (e.g., a screen capturer
309 // where the screen's content has not changed in a while), consumers may call
310 // this to request a "refresh frame" be delivered to the Client. This is used
311 // in a number of circumstances, such as:
313 // 1. An additional consumer of video frames is starting up and requires a
314 // first frame (as opposed to not receiving a frame for an indeterminate
316 // 2. A few repeats of the same frame would allow a lossy video encoder to
317 // improve the video quality of unchanging content.
319 // The default implementation is a no-op. VideoCaptureDevice implementations
320 // are not required to honor this request, especially if they do not
321 // self-pause and/or if honoring the request would cause them to exceed their
322 // configured maximum frame rate. Any VideoCaptureDevice that does self-pause,
323 // however, should provide an implementation of this method that makes
324 // reasonable attempts to honor these requests.
326 // Note: This should only be called after AllocateAndStart() and before
327 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
328 virtual void RequestRefreshFrame() {}
330 // Optionally suspends frame delivery. The VideoCaptureDevice may or may not
331 // honor this request. Thus, the caller cannot assume frame delivery will
332 // actually stop. Even if frame delivery is suspended, this might not take
333 // effect immediately.
335 // The purpose of this is to quickly place the device into a state where it's
336 // resource utilization is minimized while there are no frame consumers; and
337 // then quickly resume once a frame consumer is present.
339 // Note: This should only be called after AllocateAndStart() and before
340 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
341 virtual void MaybeSuspend() {}
343 // Resumes frame delivery, if it was suspended. If frame delivery was not
344 // suspended, this is a no-op, and frame delivery will continue.
346 // Note: This should only be called after AllocateAndStart() and before
347 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
348 virtual void Resume() {}
350 // Start/stop cropping.
352 // Non-empty |crop_id| sets (or changes) the crop-target.
353 // Empty |crop_id| reverts the capture to its original, uncropped state.
355 // |sub_capture_target_version| must be incremented by at least one for each
356 // call. By including it in frame's metadata, Viz informs Blink what was the
357 // latest invocation of cropTo() before a given frame was produced.
359 // The callback reports success/failure. It is called on an unspecified
360 // thread, it's the caller's responsibility to wrap it (i.e. via BindPostTask)
363 const base::Token& crop_id,
364 uint32_t sub_capture_target_version,
365 base::OnceCallback<void(media::mojom::ApplySubCaptureTargetResult)>
368 // Deallocates the video capturer, possibly asynchronously.
370 // This call requires the device to do the following things, eventually: put
371 // hardware into a state where other applications could use it, free the
372 // memory associated with capture, and delete the |client| pointer passed into
375 // If deallocation is done asynchronously, then the device implementation must
376 // ensure that a subsequent AllocateAndStart() operation targeting the same ID
377 // would be sequenced through the same task runner, so that deallocation
379 virtual void StopAndDeAllocate() = 0;
381 // Retrieve the photo capabilities and settings of the device (e.g. zoom
382 // levels etc). On success, invokes |callback|. On failure, drops callback
383 // without invoking it.
384 using GetPhotoStateCallback = base::OnceCallback<void(mojom::PhotoStatePtr)>;
385 virtual void GetPhotoState(GetPhotoStateCallback callback);
387 // On success, invokes |callback| with value |true|. On failure, drops
388 // callback without invoking it.
389 using SetPhotoOptionsCallback = base::OnceCallback<void(bool)>;
390 virtual void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
391 SetPhotoOptionsCallback callback);
393 // Asynchronously takes a photo, possibly reconfiguring the capture objects
394 // and/or interrupting the capture flow. Runs |callback|, if the photo was
395 // successfully taken. On failure, drops callback without invoking it.
396 // Note that |callback| may be runned on a thread different than the thread
397 // where TakePhoto() was called.
398 using TakePhotoCallback = base::OnceCallback<void(mojom::BlobPtr blob)>;
399 virtual void TakePhoto(TakePhotoCallback callback);
401 // Gets the power line frequency, either from the params if specified by the
402 // user or from the current system time zone.
403 static PowerLineFrequency GetPowerLineFrequency(
404 const VideoCaptureParams& params);
407 // Gets the power line frequency from the current system time zone if this is
408 // defined, otherwise returns 0.
409 static PowerLineFrequency GetPowerLineFrequencyForLocation();
412 VideoCaptureFrameDropReason ConvertReservationFailureToFrameDropReason(
413 VideoCaptureDevice::Client::ReserveResult reserve_result);
417 #endif // MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_