1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file contains an implementation of VideoDecodeAccelerator
6 // that utilizes hardware video decoders, which expose Video4Linux 2 API
7 // (http://linuxtv.org/downloads/v4l-dvb-apis/).
9 #ifndef CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_
10 #define CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_
15 #include "base/callback_forward.h"
16 #include "base/memory/linked_ptr.h"
17 #include "base/memory/scoped_ptr.h"
18 #include "base/synchronization/waitable_event.h"
19 #include "base/threading/thread.h"
20 #include "content/common/content_export.h"
21 #include "content/common/gpu/media/v4l2_video_device.h"
22 #include "content/common/gpu/media/video_decode_accelerator_impl.h"
23 #include "media/base/limits.h"
24 #include "media/base/video_decoder_config.h"
25 #include "media/video/picture.h"
26 #include "ui/gfx/size.h"
27 #include "ui/gl/gl_bindings.h"
30 class MessageLoopProxy;
38 // This class handles video accelerators directly through a V4L2 device exported
39 // by the hardware blocks.
41 // The threading model of this class is driven by the fact that it needs to
42 // interface two fundamentally different event queues -- the one Chromium
43 // provides through MessageLoop, and the one driven by the V4L2 devices which
44 // is waited on with epoll(). There are three threads involved in this class:
46 // * The child thread, which is the main GPU process thread which calls the
47 // media::VideoDecodeAccelerator entry points. Calls from this thread
48 // generally do not block (with the exception of Initialize() and Destroy()).
49 // They post tasks to the decoder_thread_, which actually services the task
50 // and calls back when complete through the
51 // media::VideoDecodeAccelerator::Client interface.
52 // * The decoder_thread_, owned by this class. It services API tasks, through
53 // the *Task() routines, as well as V4L2 device events, through
54 // ServiceDeviceTask(). Almost all state modification is done on this thread
55 // (this doesn't include buffer (re)allocation sequence, see below).
56 // * The device_poll_thread_, owned by this class. All it does is epoll() on
57 // the V4L2 in DevicePollTask() and schedule a ServiceDeviceTask() on the
58 // decoder_thread_ when something interesting happens.
59 // TODO(sheu): replace this thread with an TYPE_IO decoder_thread_.
61 // Note that this class has (almost) no locks, apart from the pictures_assigned_
62 // WaitableEvent. Everything (apart from buffer (re)allocation) is serviced on
63 // the decoder_thread_, so there are no synchronization issues.
64 // ... well, there are, but it's a matter of getting messages posted in the
65 // right order, not fiddling with locks.
66 // Buffer creation is a two-step process that is serviced partially on the
67 // Child thread, because we need to wait for the client to provide textures
68 // for the buffers we allocate. We cannot keep the decoder thread running while
69 // the client allocates Pictures for us, because we need to REQBUFS first to get
70 // the required number of output buffers from the device and that cannot be done
71 // unless we free the previous set of buffers, leaving the decoding in a
72 // inoperable state for the duration of the wait for Pictures. So to prevent
73 // subtle races (esp. if we get Reset() in the meantime), we block the decoder
74 // thread while we wait for AssignPictureBuffers from the client.
75 class CONTENT_EXPORT V4L2VideoDecodeAccelerator
76 : public VideoDecodeAcceleratorImpl {
78 V4L2VideoDecodeAccelerator(
79 EGLDisplay egl_display,
81 const base::WeakPtr<Client>& io_client_,
82 const base::Callback<bool(void)>& make_context_current,
83 scoped_ptr<V4L2Device> device,
84 const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy);
85 virtual ~V4L2VideoDecodeAccelerator();
87 // media::VideoDecodeAccelerator implementation.
88 // Note: Initialize() and Destroy() are synchronous.
89 virtual bool Initialize(media::VideoCodecProfile profile) OVERRIDE;
90 virtual void Decode(const media::BitstreamBuffer& bitstream_buffer) OVERRIDE;
91 virtual void AssignPictureBuffers(
92 const std::vector<media::PictureBuffer>& buffers) OVERRIDE;
93 virtual void ReusePictureBuffer(int32 picture_buffer_id) OVERRIDE;
94 virtual void Flush() OVERRIDE;
95 virtual void Reset() OVERRIDE;
96 virtual void Destroy() OVERRIDE;
98 // VideoDecodeAcceleratorImpl implementation.
99 virtual bool CanDecodeOnIOThread() OVERRIDE;
102 // These are rather subjectively tuned.
104 kInputBufferCount = 8,
105 // TODO(posciak): determine input buffer size based on level limits.
106 // See http://crbug.com/255116.
107 kInputBufferMaxSize = 1024 * 1024,
108 // Number of output buffers to use for each VDA stage above what's required
109 // by the decoder (e.g. DPB size, in H264). We need
110 // media::limits::kMaxVideoFrames to fill up the GpuVideoDecode pipeline,
111 // and +1 for a frame in transit.
112 kDpbOutputBufferExtraCount = media::limits::kMaxVideoFrames + 1,
115 // Internal state of the decoder.
117 kUninitialized, // Initialize() not yet called.
118 kInitialized, // Initialize() returned true; ready to start decoding.
119 kDecoding, // DecodeBufferInitial() successful; decoding frames.
120 kResetting, // Presently resetting.
121 kAfterReset, // After Reset(), ready to start decoding again.
122 kChangingResolution, // Performing resolution change, all remaining
123 // pre-change frames decoded and processed.
124 kError, // Error in kDecoding state.
128 kFlushBufferId = -2 // Buffer id for flush buffer, queued by FlushTask().
131 // Auto-destruction reference for BitstreamBuffer, for message-passing from
132 // Decode() to DecodeTask().
133 struct BitstreamBufferRef;
135 // Auto-destruction reference for EGLSync (for message-passing).
136 struct EGLSyncKHRRef;
138 // Record for decoded pictures that can be sent to PictureReady.
139 struct PictureRecord;
141 // Record for input buffers.
145 bool at_device; // held by device.
146 void* address; // mmap() address.
147 size_t length; // mmap() length.
148 off_t bytes_used; // bytes filled in the mmap() segment.
149 int32 input_id; // triggering input_id as given to Decode().
152 // Record for output buffers.
153 struct OutputRecord {
156 bool at_device; // held by device.
157 bool at_client; // held by client.
158 int fds[2]; // file descriptors for each plane.
159 EGLImageKHR egl_image; // EGLImageKHR for the output buffer.
160 EGLSyncKHR egl_sync; // sync the compositor's use of the EGLImage.
161 int32 picture_id; // picture buffer id as returned to PictureReady().
162 bool cleared; // Whether the texture is cleared and safe to render
163 // from. See TextureManager for details.
167 // Decoding tasks, to be run on decode_thread_.
170 // Enqueue a BitstreamBuffer to decode. This will enqueue a buffer to the
171 // decoder_input_queue_, then queue a DecodeBufferTask() to actually decode
173 void DecodeTask(const media::BitstreamBuffer& bitstream_buffer);
175 // Decode from the buffers queued in decoder_input_queue_. Calls
176 // DecodeBufferInitial() or DecodeBufferContinue() as appropriate.
177 void DecodeBufferTask();
178 // Advance to the next fragment that begins a frame.
179 bool AdvanceFrameFragment(const uint8* data, size_t size, size_t* endpos);
180 // Schedule another DecodeBufferTask() if we're behind.
181 void ScheduleDecodeBufferTaskIfNeeded();
183 // Return true if we should continue to schedule DecodeBufferTask()s after
184 // completion. Store the amount of input actually consumed in |endpos|.
185 bool DecodeBufferInitial(const void* data, size_t size, size_t* endpos);
186 bool DecodeBufferContinue(const void* data, size_t size);
188 // Accumulate data for the next frame to decode. May return false in
189 // non-error conditions; for example when pipeline is full and should be
191 bool AppendToInputFrame(const void* data, size_t size);
192 // Flush data for one decoded frame.
193 bool FlushInputFrame();
195 // Service I/O on the V4L2 devices. This task should only be scheduled from
196 // DevicePollTask(). If |event_pending| is true, one or more events
197 // on file descriptor are pending.
198 void ServiceDeviceTask(bool event_pending);
199 // Handle the various device queues.
202 // Handle incoming events.
203 void DequeueEvents();
204 // Enqueue a buffer on the corresponding queue.
205 bool EnqueueInputRecord();
206 bool EnqueueOutputRecord();
208 // Process a ReusePictureBuffer() API call. The API call create an EGLSync
209 // object on the main (GPU process) thread; we will record this object so we
210 // can wait on it before reusing the buffer.
211 void ReusePictureBufferTask(int32 picture_buffer_id,
212 scoped_ptr<EGLSyncKHRRef> egl_sync_ref);
214 // Flush() task. Child thread should not submit any more buffers until it
215 // receives the NotifyFlushDone callback. This task will schedule an empty
216 // BitstreamBufferRef (with input_id == kFlushBufferId) to perform the flush.
218 // Notify the client of a flush completion, if required. This should be
219 // called any time a relevant queue could potentially be emptied: see
220 // function definition.
221 void NotifyFlushDoneIfNeeded();
223 // Reset() task. This task will schedule a ResetDoneTask() that will send
224 // the NotifyResetDone callback, then set the decoder state to kResetting so
225 // that all intervening tasks will drain.
227 // ResetDoneTask() will set the decoder state back to kAfterReset, so
228 // subsequent decoding can continue.
229 void ResetDoneTask();
231 // Device destruction task.
234 // Attempt to start/stop device_poll_thread_.
235 bool StartDevicePoll();
236 // If |keep_input_state| is true, don't reset input state; used during
237 // resolution change.
238 bool StopDevicePoll(bool keep_input_state);
240 void StartResolutionChangeIfNeeded();
241 void FinishResolutionChange();
243 // Try to get output format, detected after parsing the beginning
244 // of the stream. Sets |again| to true if more parsing is needed.
245 bool GetFormatInfo(struct v4l2_format* format, bool* again);
246 // Create output buffers for the given |format|.
247 bool CreateBuffersForFormat(const struct v4l2_format& format);
250 // Device tasks, to be run on device_poll_thread_.
254 void DevicePollTask(bool poll_device);
257 // Safe from any thread.
260 // Error notification (using PostTask() to child thread, if necessary).
261 void NotifyError(Error error);
263 // Set the decoder_thread_ state (using PostTask to decoder thread, if
265 void SetDecoderState(State state);
268 // Other utility functions. Called on decoder_thread_, unless
269 // decoder_thread_ is not yet started, in which case the child thread can call
270 // these (e.g. in Initialize() or Destroy()).
273 // Create the buffers we need.
274 bool CreateInputBuffers();
275 bool CreateOutputBuffers();
278 // Methods run on child thread.
282 void DestroyInputBuffers();
283 // In contrast to DestroyInputBuffers, which is called only from destructor,
284 // we call DestroyOutputBuffers also during playback, on resolution change.
285 // Even if anything fails along the way, we still want to go on and clean
286 // up as much as possible, so return false if this happens, so that the
287 // caller can error out on resolution change.
288 bool DestroyOutputBuffers();
289 void ResolutionChangeDestroyBuffers();
291 // Send decoded pictures to PictureReady.
292 void SendPictureReady();
294 // Callback that indicates a picture has been cleared.
295 void PictureCleared();
297 // Our original calling message loop for the child thread.
298 scoped_refptr<base::MessageLoopProxy> child_message_loop_proxy_;
300 // Message loop of the IO thread.
301 scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
303 // WeakPtr<> pointing to |this| for use in posting tasks from the decoder or
304 // device worker threads back to the child thread. Because the worker threads
305 // are members of this class, any task running on those threads is guaranteed
306 // that this object is still alive. As a result, tasks posted from the child
307 // thread to the decoder or device thread should use base::Unretained(this),
308 // and tasks posted the other way should use |weak_this_|.
309 base::WeakPtr<V4L2VideoDecodeAccelerator> weak_this_;
311 // To expose client callbacks from VideoDecodeAccelerator.
312 // NOTE: all calls to these objects *MUST* be executed on
313 // child_message_loop_proxy_.
314 base::WeakPtrFactory<Client> client_ptr_factory_;
315 base::WeakPtr<Client> client_;
316 // Callbacks to |io_client_| must be executed on |io_message_loop_proxy_|.
317 base::WeakPtr<Client> io_client_;
320 // Decoder state, owned and operated by decoder_thread_.
321 // Before decoder_thread_ has started, the decoder state is managed by
322 // the child (main) thread. After decoder_thread_ has started, the decoder
323 // thread should be the only one managing these.
326 // This thread services tasks posted from the VDA API entry points by the
327 // child thread and device service callbacks posted from the device thread.
328 base::Thread decoder_thread_;
329 // Decoder state machine state.
330 State decoder_state_;
331 // BitstreamBuffer we're presently reading.
332 scoped_ptr<BitstreamBufferRef> decoder_current_bitstream_buffer_;
333 // The V4L2Device this class is operating upon.
334 scoped_ptr<V4L2Device> device_;
335 // FlushTask() and ResetTask() should not affect buffers that have been
336 // queued afterwards. For flushing or resetting the pipeline then, we will
337 // delay these buffers until after the flush or reset completes.
338 int decoder_delay_bitstream_buffer_id_;
339 // Input buffer we're presently filling.
340 int decoder_current_input_buffer_;
341 // We track the number of buffer decode tasks we have scheduled, since each
342 // task execution should complete one buffer. If we fall behind (due to
343 // resource backpressure, etc.), we'll have to schedule more to catch up.
344 int decoder_decode_buffer_tasks_scheduled_;
345 // Picture buffers held by the client.
346 int decoder_frames_at_client_;
348 bool decoder_flushing_;
349 // Got a notification from driver that it reached resolution change point
351 bool resolution_change_pending_;
352 // Got a reset request while we were performing resolution change.
353 bool resolution_change_reset_pending_;
354 // Input queue for decoder_thread_: BitstreamBuffers in.
355 std::queue<linked_ptr<BitstreamBufferRef> > decoder_input_queue_;
356 // For H264 decode, hardware requires that we send it frame-sized chunks.
357 // We'll need to parse the stream.
358 scoped_ptr<media::H264Parser> decoder_h264_parser_;
359 // Set if the decoder has a pending incomplete frame in an input buffer.
360 bool decoder_partial_frame_pending_;
363 // Hardware state and associated queues. Since decoder_thread_ services
364 // the hardware, decoder_thread_ owns these too.
365 // output_buffer_map_ and free_output_buffers_ are an exception during the
366 // buffer (re)allocation sequence, when the decoder_thread_ is blocked briefly
367 // while the Child thread manipulates them.
370 // Completed decode buffers.
371 std::queue<int> input_ready_queue_;
373 // Input buffer state.
374 bool input_streamon_;
375 // Input buffers enqueued to device.
376 int input_buffer_queued_count_;
377 // Input buffers ready to use, as a LIFO since we don't care about ordering.
378 std::vector<int> free_input_buffers_;
379 // Mapping of int index to input buffer record.
380 std::vector<InputRecord> input_buffer_map_;
382 // Output buffer state.
383 bool output_streamon_;
384 // Output buffers enqueued to device.
385 int output_buffer_queued_count_;
386 // Output buffers ready to use, as a FIFO since we want oldest-first to hide
387 // synchronization latency with GL.
388 std::queue<int> free_output_buffers_;
389 // Mapping of int index to output buffer record.
390 std::vector<OutputRecord> output_buffer_map_;
391 // Output pixel format.
392 uint32 output_buffer_pixelformat_;
393 // Required size of DPB for decoding.
394 int output_dpb_size_;
396 // Pictures that are ready but not sent to PictureReady yet.
397 std::queue<PictureRecord> pending_picture_ready_;
399 // The number of pictures that are sent to PictureReady and will be cleared.
400 int picture_clearing_count_;
402 // Used by the decoder thread to wait for AssignPictureBuffers to arrive
403 // to avoid races with potential Reset requests.
404 base::WaitableEvent pictures_assigned_;
406 // Output picture size.
407 gfx::Size frame_buffer_size_;
410 // The device polling thread handles notifications of V4L2 device changes.
414 base::Thread device_poll_thread_;
417 // Other state, held by the child (main) thread.
420 // Make our context current before running any EGL entry points.
421 base::Callback<bool(void)> make_context_current_;
424 EGLDisplay egl_display_;
426 // The codec we'll be decoding for.
427 media::VideoCodecProfile video_profile_;
429 DISALLOW_COPY_AND_ASSIGN(V4L2VideoDecodeAccelerator);
432 } // namespace content
434 #endif // CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_