1 // Copyright 2014 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/pepper/video_decoder_shim.h"
8 #include <GLES2/gl2ext.h>
9 #include <GLES2/gl2extchromium.h>
14 #include "base/bind.h"
15 #include "base/callback_helpers.h"
16 #include "base/check_op.h"
17 #include "base/containers/queue.h"
18 #include "base/location.h"
19 #include "base/memory/ref_counted.h"
20 #include "base/notreached.h"
21 #include "base/numerics/safe_conversions.h"
22 #include "base/task/single_thread_task_runner.h"
23 #include "base/threading/thread_task_runner_handle.h"
24 #include "content/public/renderer/render_thread.h"
25 #include "content/renderer/pepper/pepper_video_decoder_host.h"
26 #include "content/renderer/render_thread_impl.h"
27 #include "gpu/command_buffer/client/raster_interface.h"
28 #include "media/base/cdm_context.h"
29 #include "media/base/decoder_buffer.h"
30 #include "media/base/limits.h"
31 #include "media/base/media_util.h"
32 #include "media/base/status.h"
33 #include "media/base/video_decoder.h"
34 #include "media/filters/ffmpeg_video_decoder.h"
35 #include "media/filters/vpx_video_decoder.h"
36 #include "media/media_buildflags.h"
37 #include "media/renderers/video_frame_yuv_converter.h"
38 #include "media/video/picture.h"
39 #include "media/video/video_decode_accelerator.h"
40 #include "ppapi/c/pp_errors.h"
41 #include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
42 #include "third_party/skia/include/gpu/GrTypes.h"
48 bool IsCodecSupported(media::VideoCodec codec) {
49 #if BUILDFLAG(ENABLE_LIBVPX)
50 if (codec == media::VideoCodec::kVP9)
54 #if BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
55 if (media::FFmpegVideoDecoder::IsCodecSupported(codec))
63 struct VideoDecoderShim::PendingDecode {
64 PendingDecode(uint32_t decode_id,
65 const scoped_refptr<media::DecoderBuffer>& buffer);
68 const uint32_t decode_id;
69 const scoped_refptr<media::DecoderBuffer> buffer;
72 VideoDecoderShim::PendingDecode::PendingDecode(
74 const scoped_refptr<media::DecoderBuffer>& buffer)
75 : decode_id(decode_id), buffer(buffer) {
78 VideoDecoderShim::PendingDecode::~PendingDecode() {
81 struct VideoDecoderShim::PendingFrame {
82 explicit PendingFrame(uint32_t decode_id);
83 PendingFrame(uint32_t decode_id, scoped_refptr<media::VideoFrame> frame);
85 // This could be expensive to copy, so guard against that.
86 PendingFrame(const PendingFrame&) = delete;
87 PendingFrame& operator=(const PendingFrame&) = delete;
91 const uint32_t decode_id;
92 scoped_refptr<media::VideoFrame> video_frame;
95 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
96 : decode_id(decode_id) {
99 VideoDecoderShim::PendingFrame::PendingFrame(
101 scoped_refptr<media::VideoFrame> frame)
102 : decode_id(decode_id), video_frame(std::move(frame)) {}
104 VideoDecoderShim::PendingFrame::~PendingFrame() {
107 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
108 // calls from the VideoDecodeShim on the main thread and sending results back.
109 // This class is constructed on the main thread, but used and destructed on the
111 class VideoDecoderShim::DecoderImpl {
113 explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
116 void Initialize(media::VideoDecoderConfig config);
117 void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
122 void OnInitDone(media::DecoderStatus status);
124 void OnDecodeComplete(media::DecoderStatus status);
125 void OnOutputComplete(scoped_refptr<media::VideoFrame> frame);
126 void OnResetComplete();
128 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
129 base::WeakPtr<VideoDecoderShim> shim_;
130 media::NullMediaLog media_log_;
131 std::unique_ptr<media::VideoDecoder> decoder_;
132 bool initialized_ = false;
133 scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
134 // Queue of decodes waiting for the decoder.
135 using PendingDecodeQueue = base::queue<PendingDecode>;
136 PendingDecodeQueue pending_decodes_;
137 bool awaiting_decoder_ = false;
138 // VideoDecoder returns pictures without information about the decode buffer
139 // that generated it, but VideoDecoder implementations used in this class
140 // (media::FFmpegVideoDecoder and media::VpxVideoDecoder) always generate
141 // corresponding frames before decode is finished. |decode_id_| is used to
142 // store id of the current buffer while Decode() call is pending.
143 uint32_t decode_id_ = 0;
145 base::WeakPtrFactory<DecoderImpl> weak_ptr_factory_{this};
148 VideoDecoderShim::DecoderImpl::DecoderImpl(
149 const base::WeakPtr<VideoDecoderShim>& proxy)
150 : shim_(proxy), main_task_runner_(base::ThreadTaskRunnerHandle::Get()) {}
152 VideoDecoderShim::DecoderImpl::~DecoderImpl() {
153 DCHECK(pending_decodes_.empty());
156 void VideoDecoderShim::DecoderImpl::Initialize(
157 media::VideoDecoderConfig config) {
159 #if BUILDFLAG(ENABLE_LIBVPX) || BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
160 #if BUILDFLAG(ENABLE_LIBVPX)
161 if (config.codec() == media::VideoCodec::kVP9) {
162 decoder_ = std::make_unique<media::VpxVideoDecoder>();
164 #endif // BUILDFLAG(ENABLE_LIBVPX)
165 #if BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
167 std::unique_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder(
168 new media::FFmpegVideoDecoder(&media_log_));
169 ffmpeg_video_decoder->set_decode_nalus(true);
170 decoder_ = std::move(ffmpeg_video_decoder);
172 #endif // BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
173 // VpxVideoDecoder and FFmpegVideoDecoder support only one pending Decode()
175 DCHECK_EQ(decoder_->GetMaxDecodeRequests(), 1);
177 decoder_->Initialize(
178 config, true /* low_delay */, nullptr,
179 base::BindOnce(&VideoDecoderShim::DecoderImpl::OnInitDone,
180 weak_ptr_factory_.GetWeakPtr()),
181 base::BindRepeating(&VideoDecoderShim::DecoderImpl::OnOutputComplete,
182 weak_ptr_factory_.GetWeakPtr()),
183 base::NullCallback());
185 OnInitDone(media::DecoderStatus::Codes::kUnsupportedCodec);
186 #endif // BUILDFLAG(ENABLE_LIBVPX) || BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
189 void VideoDecoderShim::DecoderImpl::Decode(
191 scoped_refptr<media::DecoderBuffer> buffer) {
193 pending_decodes_.push(PendingDecode(decode_id, buffer));
197 void VideoDecoderShim::DecoderImpl::Reset() {
199 // Abort all pending decodes.
200 while (!pending_decodes_.empty()) {
201 const PendingDecode& decode = pending_decodes_.front();
202 std::unique_ptr<PendingFrame> pending_frame(
203 new PendingFrame(decode.decode_id));
204 main_task_runner_->PostTask(
205 FROM_HERE, base::BindOnce(&VideoDecoderShim::OnDecodeComplete, shim_,
206 PP_OK, decode.decode_id));
207 pending_decodes_.pop();
209 // Don't need to call Reset() if the |decoder_| hasn't been initialized.
216 base::BindOnce(&VideoDecoderShim::DecoderImpl::OnResetComplete,
217 weak_ptr_factory_.GetWeakPtr()));
220 void VideoDecoderShim::DecoderImpl::Stop() {
222 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
224 while (!pending_decodes_.empty())
225 pending_decodes_.pop();
227 // This instance is deleted once we exit this scope.
230 void VideoDecoderShim::DecoderImpl::OnInitDone(media::DecoderStatus status) {
231 if (!status.is_ok()) {
232 main_task_runner_->PostTask(
234 base::BindOnce(&VideoDecoderShim::OnInitializeFailed, shim_));
242 void VideoDecoderShim::DecoderImpl::DoDecode() {
243 if (!initialized_ || pending_decodes_.empty() || awaiting_decoder_)
246 awaiting_decoder_ = true;
247 const PendingDecode& decode = pending_decodes_.front();
248 decode_id_ = decode.decode_id;
251 base::BindOnce(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
252 weak_ptr_factory_.GetWeakPtr()));
253 pending_decodes_.pop();
256 void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
257 media::DecoderStatus status) {
258 DCHECK(awaiting_decoder_);
259 awaiting_decoder_ = false;
262 switch (status.code()) {
263 case media::DecoderStatus::Codes::kOk:
264 case media::DecoderStatus::Codes::kAborted:
268 LOG(ERROR) << "Decode Error occurred";
269 result = PP_ERROR_RESOURCE_FAILED;
273 main_task_runner_->PostTask(
274 FROM_HERE, base::BindOnce(&VideoDecoderShim::OnDecodeComplete, shim_,
275 result, decode_id_));
280 void VideoDecoderShim::DecoderImpl::OnOutputComplete(
281 scoped_refptr<media::VideoFrame> frame) {
282 // Software decoders are expected to generated frames only when a Decode()
284 DCHECK(awaiting_decoder_);
286 std::unique_ptr<PendingFrame> pending_frame;
287 if (!frame->metadata().end_of_stream) {
289 std::make_unique<PendingFrame>(decode_id_, std::move(frame));
291 pending_frame = std::make_unique<PendingFrame>(decode_id_);
294 main_task_runner_->PostTask(
295 FROM_HERE, base::BindOnce(&VideoDecoderShim::OnOutputComplete, shim_,
296 std::move(pending_frame)));
299 void VideoDecoderShim::DecoderImpl::OnResetComplete() {
300 main_task_runner_->PostTask(
301 FROM_HERE, base::BindOnce(&VideoDecoderShim::OnResetComplete, shim_));
304 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host,
305 uint32_t texture_pool_size)
306 : state_(UNINITIALIZED),
309 RenderThreadImpl::current()->GetMediaThreadTaskRunner()),
311 RenderThreadImpl::current()->SharedMainThreadContextProvider()),
312 texture_pool_size_(texture_pool_size),
313 num_pending_decodes_(0) {
315 DCHECK(media_task_runner_.get());
316 DCHECK(context_provider_.get());
317 decoder_impl_ = std::make_unique<DecoderImpl>(weak_ptr_factory_.GetWeakPtr());
320 VideoDecoderShim::~VideoDecoderShim() {
321 DCHECK(RenderThreadImpl::current());
322 texture_mailbox_map_.clear();
324 FlushCommandBuffer();
326 weak_ptr_factory_.InvalidateWeakPtrs();
327 // No more callbacks from the delegate will be received now.
329 // The callback now holds the only reference to the DecoderImpl, which will be
330 // deleted when Stop completes.
331 media_task_runner_->PostTask(
332 FROM_HERE, base::BindOnce(&VideoDecoderShim::DecoderImpl::Stop,
333 base::Owned(decoder_impl_.release())));
336 bool VideoDecoderShim::Initialize(const Config& vda_config, Client* client) {
337 DCHECK_EQ(client, host_);
338 DCHECK(RenderThreadImpl::current());
339 DCHECK_EQ(state_, UNINITIALIZED);
341 if (vda_config.is_encrypted()) {
342 NOTREACHED() << "Encrypted streams are not supported";
346 media::VideoCodec codec = media::VideoCodec::kUnknown;
347 if (vda_config.profile <= media::H264PROFILE_MAX)
348 codec = media::VideoCodec::kH264;
349 else if (vda_config.profile <= media::VP8PROFILE_MAX)
350 codec = media::VideoCodec::kVP8;
351 else if (vda_config.profile <= media::VP9PROFILE_MAX)
352 codec = media::VideoCodec::kVP9;
353 DCHECK_NE(codec, media::VideoCodec::kUnknown);
355 if (!IsCodecSupported(codec))
358 media::VideoDecoderConfig video_decoder_config(
359 codec, vda_config.profile,
360 media::VideoDecoderConfig::AlphaMode::kIsOpaque, media::VideoColorSpace(),
361 media::kNoTransformation,
362 gfx::Size(32, 24), // Small sizes that won't fail.
363 gfx::Rect(32, 24), gfx::Size(32, 24),
364 // TODO(bbudge): Verify extra data isn't needed.
365 media::EmptyExtraData(), media::EncryptionScheme::kUnencrypted);
367 media_task_runner_->PostTask(
368 FROM_HERE, base::BindOnce(&VideoDecoderShim::DecoderImpl::Initialize,
369 base::Unretained(decoder_impl_.get()),
370 video_decoder_config));
374 // Return success, even though we are asynchronous, to mimic
375 // media::VideoDecodeAccelerator.
379 void VideoDecoderShim::Decode(media::BitstreamBuffer bitstream_buffer) {
380 DCHECK(RenderThreadImpl::current());
381 DCHECK_EQ(state_, DECODING);
383 // We need the address of the shared memory, so we can copy the buffer.
384 const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id());
387 media_task_runner_->PostTask(
390 &VideoDecoderShim::DecoderImpl::Decode,
391 base::Unretained(decoder_impl_.get()), bitstream_buffer.id(),
392 media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
393 num_pending_decodes_++;
396 void VideoDecoderShim::AssignPictureBuffers(
397 const std::vector<media::PictureBuffer>& buffers) {
398 DCHECK(RenderThreadImpl::current());
399 DCHECK_NE(state_, UNINITIALIZED);
400 if (buffers.empty()) {
404 std::vector<gpu::Mailbox> mailboxes = host_->TakeMailboxes();
405 DCHECK_EQ(buffers.size(), mailboxes.size());
406 GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
407 for (uint32_t i = 0; i < num_textures; i++) {
408 DCHECK_EQ(1u, buffers[i].client_texture_ids().size());
409 // Map the plugin texture id to the mailbox
410 uint32_t plugin_texture_id = buffers[i].client_texture_ids()[0];
411 texture_mailbox_map_[plugin_texture_id] = mailboxes[i];
412 available_textures_.insert(plugin_texture_id);
417 void VideoDecoderShim::ReusePictureBuffer(int32_t picture_buffer_id) {
418 DCHECK(RenderThreadImpl::current());
419 uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
420 if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
421 DismissTexture(texture_id);
422 } else if (texture_mailbox_map_.find(texture_id) !=
423 texture_mailbox_map_.end()) {
424 available_textures_.insert(texture_id);
431 void VideoDecoderShim::Flush() {
432 DCHECK(RenderThreadImpl::current());
433 DCHECK_EQ(state_, DECODING);
437 void VideoDecoderShim::Reset() {
438 DCHECK(RenderThreadImpl::current());
439 DCHECK_EQ(state_, DECODING);
441 media_task_runner_->PostTask(
442 FROM_HERE, base::BindOnce(&VideoDecoderShim::DecoderImpl::Reset,
443 base::Unretained(decoder_impl_.get())));
446 void VideoDecoderShim::Destroy() {
450 void VideoDecoderShim::OnInitializeFailed() {
451 DCHECK(RenderThreadImpl::current());
454 host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
457 void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) {
458 DCHECK(RenderThreadImpl::current());
461 if (result == PP_ERROR_RESOURCE_FAILED) {
462 host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
466 num_pending_decodes_--;
467 completed_decodes_.push(decode_id);
469 // If frames are being queued because we're out of textures, don't notify
470 // the host that decode has completed. This exerts "back pressure" to keep
471 // the host from sending buffers that will cause pending_frames_ to grow.
472 if (pending_frames_.empty())
473 NotifyCompletedDecodes();
476 void VideoDecoderShim::OnOutputComplete(std::unique_ptr<PendingFrame> frame) {
477 DCHECK(RenderThreadImpl::current());
480 if (frame->video_frame) {
481 if (texture_size_ != frame->video_frame->coded_size()) {
482 // If the size has changed, all current textures must be dismissed. Add
483 // all textures to |textures_to_dismiss_| and dismiss any that aren't in
484 // use by the plugin. We will dismiss the rest as they are recycled.
485 for (IdToMailboxMap::const_iterator it = texture_mailbox_map_.begin();
486 it != texture_mailbox_map_.end(); ++it) {
487 textures_to_dismiss_.insert(it->first);
489 for (auto it = available_textures_.begin();
490 it != available_textures_.end(); ++it) {
493 available_textures_.clear();
494 FlushCommandBuffer();
496 host_->ProvidePictureBuffers(texture_pool_size_, media::PIXEL_FORMAT_ARGB,
497 1, frame->video_frame->coded_size(),
499 texture_size_ = frame->video_frame->coded_size();
502 pending_frames_.push(std::move(frame));
507 void VideoDecoderShim::SendPictures() {
508 DCHECK(RenderThreadImpl::current());
510 while (!pending_frames_.empty() && !available_textures_.empty()) {
511 const std::unique_ptr<PendingFrame>& frame = pending_frames_.front();
513 auto it = available_textures_.begin();
514 uint32_t texture_id = *it;
515 available_textures_.erase(it);
517 gpu::MailboxHolder destination_holder;
518 destination_holder.mailbox = texture_mailbox_map_[texture_id];
519 destination_holder.texture_target = GL_TEXTURE_2D;
520 media::VideoFrameYUVConverter::ConvertYUVVideoFrameNoCaching(
521 frame->video_frame.get(), context_provider_.get(), destination_holder);
522 host_->PictureReady(media::Picture(texture_id, frame->decode_id,
523 frame->video_frame->visible_rect(),
524 gfx::ColorSpace(), false));
525 pending_frames_.pop();
528 FlushCommandBuffer();
530 if (pending_frames_.empty()) {
531 // If frames aren't backing up, notify the host of any completed decodes so
532 // it can send more buffers.
533 NotifyCompletedDecodes();
535 if (state_ == FLUSHING && !num_pending_decodes_) {
537 host_->NotifyFlushDone();
542 void VideoDecoderShim::OnResetComplete() {
543 DCHECK(RenderThreadImpl::current());
546 while (!pending_frames_.empty())
547 pending_frames_.pop();
548 NotifyCompletedDecodes();
550 // Dismiss any old textures now.
551 while (!textures_to_dismiss_.empty())
552 DismissTexture(*textures_to_dismiss_.begin());
555 host_->NotifyResetDone();
558 void VideoDecoderShim::NotifyCompletedDecodes() {
559 while (!completed_decodes_.empty()) {
560 host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front());
561 completed_decodes_.pop();
565 void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
567 textures_to_dismiss_.erase(texture_id);
568 DCHECK(texture_mailbox_map_.find(texture_id) != texture_mailbox_map_.end());
569 texture_mailbox_map_.erase(texture_id);
570 host_->DismissPictureBuffer(texture_id);
573 void VideoDecoderShim::FlushCommandBuffer() {
574 context_provider_->RasterInterface()->Flush();
577 } // namespace content