1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // VideoCaptureDevice is the abstract base class for realizing video capture
6 // device support in Chromium. It provides the interface for OS dependent
8 // The class is created and functions are invoked on a thread owned by
9 // VideoCaptureManager. Capturing is done on other threads, depending on the OS
10 // specific implementation.
12 #ifndef MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
13 #define MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_
22 #include "base/callback.h"
23 #include "base/files/file.h"
24 #include "base/logging.h"
25 #include "base/memory/ref_counted.h"
26 #include "base/single_thread_task_runner.h"
27 #include "base/time/time.h"
28 #include "build/build_config.h"
29 #include "media/base/video_frame.h"
30 #include "media/capture/capture_export.h"
31 #include "media/capture/mojom/image_capture.mojom.h"
32 #include "media/capture/video/video_capture_buffer_handle.h"
33 #include "media/capture/video/video_capture_device_descriptor.h"
34 #include "media/capture/video_capture_types.h"
35 #include "ui/gfx/gpu_memory_buffer.h"
43 class CAPTURE_EXPORT VideoFrameConsumerFeedbackObserver {
45 virtual ~VideoFrameConsumerFeedbackObserver() {}
47 // During processing of a video frame, consumers may report back their
48 // utilization level to the source device. The device may use this information
49 // to adjust the rate of data it pushes out. Values are interpreted as
51 // Less than 0.0 is meaningless and should be ignored. 1.0 indicates a
52 // maximum sustainable utilization. Greater than 1.0 indicates the consumer
53 // is likely to stall or drop frames if the data volume is not reduced.
55 // Example: In a system that encodes and transmits video frames over the
56 // network, this value can be used to indicate whether sufficient CPU
57 // is available for encoding and/or sufficient bandwidth is available for
58 // transmission over the network. The maximum of the two utilization
59 // measurements would be used as feedback.
61 // The parameter |frame_feedback_id| must match a |frame_feedback_id|
62 // previously sent out by the VideoCaptureDevice we are giving feedback about.
63 // It is used to indicate which particular frame the reported utilization
65 virtual void OnUtilizationReport(int frame_feedback_id, double utilization) {}
67 static constexpr double kNoUtilizationRecorded = -1.0;
70 class CAPTURE_EXPORT VideoCaptureDevice
71 : public VideoFrameConsumerFeedbackObserver {
74 // Interface defining the methods that clients of VideoCapture must have. It
75 // is actually two-in-one: clients may implement OnIncomingCapturedData() or
76 // ReserveOutputBuffer() + OnIncomingCapturedVideoFrame(), or all of them.
77 // All methods may be called as soon as AllocateAndStart() of the
78 // corresponding VideoCaptureDevice is invoked. The methods for buffer
79 // reservation and frame delivery may be called from arbitrary threads but
80 // are guaranteed to be called non-concurrently. The status reporting methods
81 // (OnStarted, OnLog, OnError) may be called concurrently.
82 class CAPTURE_EXPORT Client {
84 // Struct bundling several parameters being passed between a
85 // VideoCaptureDevice and its VideoCaptureDevice::Client.
86 struct CAPTURE_EXPORT Buffer {
88 // Destructor-only interface for encapsulating scoped access permission to
90 class CAPTURE_EXPORT ScopedAccessPermission {
92 virtual ~ScopedAccessPermission() {}
95 class CAPTURE_EXPORT HandleProvider {
97 virtual ~HandleProvider() {}
98 virtual mojo::ScopedSharedBufferHandle GetHandleForInterProcessTransit(
100 virtual base::SharedMemoryHandle
101 GetNonOwnedSharedMemoryHandleForLegacyIPC() = 0;
102 virtual std::unique_ptr<VideoCaptureBufferHandle>
103 GetHandleForInProcessAccess() = 0;
107 Buffer(int buffer_id,
108 int frame_feedback_id,
109 std::unique_ptr<HandleProvider> handle_provider,
110 std::unique_ptr<ScopedAccessPermission> access_permission);
112 Buffer(Buffer&& other);
113 Buffer& operator=(Buffer&& other);
115 bool is_valid() const { return handle_provider != nullptr; }
118 int frame_feedback_id;
119 std::unique_ptr<HandleProvider> handle_provider;
120 std::unique_ptr<ScopedAccessPermission> access_permission;
125 // Captured a new video frame, data for which is pointed to by |data|.
127 // The format of the frame is described by |frame_format|, and is assumed to
128 // be tightly packed. This method will try to reserve an output buffer and
129 // copy from |data| into the output buffer. If no output buffer is
130 // available, the frame will be silently dropped. |reference_time| is
131 // system clock time when we detect the capture happens, it is used for
132 // Audio/Video sync, not an exact presentation time for playout, because it
133 // could contain noise. |timestamp| measures the ideal time span between the
134 // first frame in the stream and the current frame; however, the time source
135 // is determined by the platform's device driver and is often not the system
136 // clock, or even has a drift with respect to system clock.
137 // |frame_feedback_id| is an identifier that allows clients to refer back to
138 // this particular frame when reporting consumer feedback via
139 // OnConsumerReportingUtilization(). This identifier is needed because
140 // frames are consumed asynchronously and multiple frames can be "in flight"
142 virtual void OnIncomingCapturedData(const uint8_t* data,
144 const VideoCaptureFormat& frame_format,
145 int clockwise_rotation,
146 base::TimeTicks reference_time,
147 base::TimeDelta timestamp,
148 int frame_feedback_id = 0) = 0;
150 // Reserve an output buffer into which contents can be captured directly.
151 // The returned Buffer will always be allocated with a memory size suitable
152 // for holding a packed video frame with pixels of |format| format, of
153 // |dimensions| frame dimensions. It is permissible for |dimensions| to be
154 // zero; in which case the returned Buffer does not guarantee memory
155 // backing, but functions as a reservation for external input for the
156 // purposes of buffer throttling.
158 // The buffer stays reserved for use by the caller as long as it
159 // holds on to the contained |buffer_read_write_permission|.
160 virtual Buffer ReserveOutputBuffer(const gfx::Size& dimensions,
161 VideoPixelFormat format,
162 VideoPixelStorage storage,
163 int frame_feedback_id) = 0;
165 // Provides VCD::Client with a populated Buffer containing the content of
166 // the next video frame. The |buffer| must originate from an earlier call to
167 // ReserveOutputBuffer().
168 // See OnIncomingCapturedData for details of |reference_time| and
170 virtual void OnIncomingCapturedBuffer(Buffer buffer,
171 const VideoCaptureFormat& format,
172 base::TimeTicks reference_time,
173 base::TimeDelta timestamp) = 0;
175 // Extended version of OnIncomingCapturedBuffer() allowing clients to
176 // pass a custom |visible_rect| and |additional_metadata|.
177 virtual void OnIncomingCapturedBufferExt(
179 const VideoCaptureFormat& format,
180 base::TimeTicks reference_time,
181 base::TimeDelta timestamp,
182 gfx::Rect visible_rect,
183 const VideoFrameMetadata& additional_metadata) = 0;
185 // Attempts to reserve the same Buffer provided in the last call to one of
186 // the OnIncomingCapturedBufferXXX() methods. This will fail if the content
187 // of the Buffer has not been preserved, or if the |dimensions|, |format|,
188 // or |storage| disagree with how it was reserved via ReserveOutputBuffer().
189 // When this operation fails, nullptr will be returned.
190 virtual Buffer ResurrectLastOutputBuffer(const gfx::Size& dimensions,
191 VideoPixelFormat format,
192 VideoPixelStorage storage,
193 int new_frame_feedback_id) = 0;
195 // An error has occurred that cannot be handled and VideoCaptureDevice must
196 // be StopAndDeAllocate()-ed. |reason| is a text description of the error.
197 virtual void OnError(const base::Location& from_here,
198 const std::string& reason) = 0;
200 // VideoCaptureDevice requests the |message| to be logged.
201 virtual void OnLog(const std::string& message) {}
203 // Returns the current buffer pool utilization, in the range 0.0 (no buffers
204 // are in use by producers or consumers) to 1.0 (all buffers are in use).
205 virtual double GetBufferPoolUtilization() const = 0;
207 // VideoCaptureDevice reports it's successfully started.
208 virtual void OnStarted() = 0;
211 ~VideoCaptureDevice() override;
213 // Prepares the video capturer for use. StopAndDeAllocate() must be called
214 // before the object is deleted.
215 virtual void AllocateAndStart(const VideoCaptureParams& params,
216 std::unique_ptr<Client> client) = 0;
218 // In cases where the video capturer self-pauses (e.g., a screen capturer
219 // where the screen's content has not changed in a while), consumers may call
220 // this to request a "refresh frame" be delivered to the Client. This is used
221 // in a number of circumstances, such as:
223 // 1. An additional consumer of video frames is starting up and requires a
224 // first frame (as opposed to not receiving a frame for an indeterminate
226 // 2. A few repeats of the same frame would allow a lossy video encoder to
227 // improve the video quality of unchanging content.
229 // The default implementation is a no-op. VideoCaptureDevice implementations
230 // are not required to honor this request, especially if they do not
231 // self-pause and/or if honoring the request would cause them to exceed their
232 // configured maximum frame rate. Any VideoCaptureDevice that does self-pause,
233 // however, should provide an implementation of this method that makes
234 // reasonable attempts to honor these requests.
236 // Note: This should only be called after AllocateAndStart() and before
237 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
238 virtual void RequestRefreshFrame() {}
240 // Optionally suspends frame delivery. The VideoCaptureDevice may or may not
241 // honor this request. Thus, the caller cannot assume frame delivery will
242 // actually stop. Even if frame delivery is suspended, this might not take
243 // effect immediately.
245 // The purpose of this is to quickly place the device into a state where it's
246 // resource utilization is minimized while there are no frame consumers; and
247 // then quickly resume once a frame consumer is present.
249 // Note: This should only be called after AllocateAndStart() and before
250 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
251 virtual void MaybeSuspend() {}
253 // Resumes frame delivery, if it was suspended. If frame delivery was not
254 // suspended, this is a no-op, and frame delivery will continue.
256 // Note: This should only be called after AllocateAndStart() and before
257 // StopAndDeAllocate(). Otherwise, its behavior is undefined.
258 virtual void Resume() {}
260 // Deallocates the video capturer, possibly asynchronously.
262 // This call requires the device to do the following things, eventually: put
263 // hardware into a state where other applications could use it, free the
264 // memory associated with capture, and delete the |client| pointer passed into
267 // If deallocation is done asynchronously, then the device implementation must
268 // ensure that a subsequent AllocateAndStart() operation targeting the same ID
269 // would be sequenced through the same task runner, so that deallocation
271 virtual void StopAndDeAllocate() = 0;
273 // Retrieve the photo capabilities and settings of the device (e.g. zoom
274 // levels etc). On success, invokes |callback|. On failure, drops callback
275 // without invoking it.
276 using GetPhotoStateCallback = base::OnceCallback<void(mojom::PhotoStatePtr)>;
277 virtual void GetPhotoState(GetPhotoStateCallback callback);
279 // On success, invokes |callback| with value |true|. On failure, drops
280 // callback without invoking it.
281 using SetPhotoOptionsCallback = base::OnceCallback<void(bool)>;
282 virtual void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
283 SetPhotoOptionsCallback callback);
285 // Asynchronously takes a photo, possibly reconfiguring the capture objects
286 // and/or interrupting the capture flow. Runs |callback|, if the photo was
287 // successfully taken. On failure, drops callback without invoking it.
288 // Note that |callback| may be runned on a thread different than the thread
289 // where TakePhoto() was called.
290 using TakePhotoCallback = base::OnceCallback<void(mojom::BlobPtr blob)>;
291 virtual void TakePhoto(TakePhotoCallback callback);
293 // Gets the power line frequency, either from the params if specified by the
294 // user or from the current system time zone.
295 PowerLineFrequency GetPowerLineFrequency(
296 const VideoCaptureParams& params) const;
299 // Gets the power line frequency from the current system time zone if this is
300 // defined, otherwise returns 0.
301 PowerLineFrequency GetPowerLineFrequencyForLocation() const;
306 #endif // MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_DEVICE_H_