1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/pepper/video_decoder_shim.h"
8 #include <GLES2/gl2ext.h>
9 #include <GLES2/gl2extchromium.h>
11 #include "base/bind.h"
12 #include "base/numerics/safe_conversions.h"
13 #include "content/public/renderer/render_thread.h"
14 #include "content/renderer/pepper/pepper_video_decoder_host.h"
15 #include "content/renderer/render_thread_impl.h"
16 #include "gpu/command_buffer/client/gles2_implementation.h"
17 #include "media/base/decoder_buffer.h"
18 #include "media/base/limits.h"
19 #include "media/base/video_decoder.h"
20 #include "media/filters/ffmpeg_video_decoder.h"
21 #include "media/filters/vpx_video_decoder.h"
22 #include "media/video/picture.h"
23 #include "media/video/video_decode_accelerator.h"
24 #include "ppapi/c/pp_errors.h"
25 #include "third_party/libyuv/include/libyuv.h"
26 #include "webkit/common/gpu/context_provider_web_context.h"
30 struct VideoDecoderShim::PendingDecode {
31 PendingDecode(uint32_t decode_id,
32 const scoped_refptr<media::DecoderBuffer>& buffer);
35 const uint32_t decode_id;
36 const scoped_refptr<media::DecoderBuffer> buffer;
39 VideoDecoderShim::PendingDecode::PendingDecode(
41 const scoped_refptr<media::DecoderBuffer>& buffer)
42 : decode_id(decode_id), buffer(buffer) {
45 VideoDecoderShim::PendingDecode::~PendingDecode() {
48 struct VideoDecoderShim::PendingFrame {
49 explicit PendingFrame(uint32_t decode_id);
50 PendingFrame(uint32_t decode_id, const gfx::Size& size);
53 const uint32_t decode_id;
55 std::vector<uint8_t> argb_pixels;
58 // This could be expensive to copy, so guard against that.
59 DISALLOW_COPY_AND_ASSIGN(PendingFrame);
62 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
63 : decode_id(decode_id) {
66 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id,
67 const gfx::Size& size)
68 : decode_id(decode_id),
70 argb_pixels(size.width() * size.height() * 4) {
73 VideoDecoderShim::PendingFrame::~PendingFrame() {
76 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
77 // calls from the VideoDecodeShim on the main thread and sending results back.
78 // This class is constructed on the main thread, but used and destructed on the
80 class VideoDecoderShim::DecoderImpl {
82 explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
85 void Initialize(media::VideoDecoderConfig config);
86 void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
91 void OnPipelineStatus(media::PipelineStatus status);
93 void OnDecodeComplete(uint32_t decode_id, media::VideoDecoder::Status status);
94 void OnOutputComplete(const scoped_refptr<media::VideoFrame>& frame);
95 void OnResetComplete();
97 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
98 base::WeakPtr<VideoDecoderShim> shim_;
99 scoped_ptr<media::VideoDecoder> decoder_;
100 scoped_refptr<base::MessageLoopProxy> main_message_loop_;
101 // Queue of decodes waiting for the decoder.
102 typedef std::queue<PendingDecode> PendingDecodeQueue;
103 PendingDecodeQueue pending_decodes_;
104 int max_decodes_at_decoder_;
105 int num_decodes_at_decoder_;
106 // VideoDecoder returns pictures without information about the decode buffer
107 // that generated it. Save the decode_id from the last decode that completed,
108 // which is close for most decoders, which only decode one buffer at a time.
112 VideoDecoderShim::DecoderImpl::DecoderImpl(
113 const base::WeakPtr<VideoDecoderShim>& proxy)
115 main_message_loop_(base::MessageLoopProxy::current()),
116 max_decodes_at_decoder_(0),
117 num_decodes_at_decoder_(0),
121 VideoDecoderShim::DecoderImpl::~DecoderImpl() {
122 DCHECK(pending_decodes_.empty());
125 void VideoDecoderShim::DecoderImpl::Initialize(
126 media::VideoDecoderConfig config) {
128 if (config.codec() == media::kCodecVP9) {
130 new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
132 scoped_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder(
133 new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
134 ffmpeg_video_decoder->set_decode_nalus(true);
135 decoder_ = ffmpeg_video_decoder.Pass();
137 max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests();
138 // We can use base::Unretained() safely in decoder callbacks because we call
139 // VideoDecoder::Stop() before deletion. Stop() guarantees there will be no
140 // outstanding callbacks after it returns.
141 decoder_->Initialize(
143 true /* low_delay */,
144 base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus,
145 base::Unretained(this)),
146 base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete,
147 base::Unretained(this)));
150 void VideoDecoderShim::DecoderImpl::Decode(
152 scoped_refptr<media::DecoderBuffer> buffer) {
154 pending_decodes_.push(PendingDecode(decode_id, buffer));
158 void VideoDecoderShim::DecoderImpl::Reset() {
160 // Abort all pending decodes.
161 while (!pending_decodes_.empty()) {
162 const PendingDecode& decode = pending_decodes_.front();
163 scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
164 main_message_loop_->PostTask(FROM_HERE,
165 base::Bind(&VideoDecoderShim::OnDecodeComplete,
167 media::VideoDecoder::kAborted,
169 pending_decodes_.pop();
171 decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete,
172 base::Unretained(this)));
175 void VideoDecoderShim::DecoderImpl::Stop() {
177 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
179 while (!pending_decodes_.empty())
180 pending_decodes_.pop();
182 // This instance is deleted once we exit this scope.
185 void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
186 media::PipelineStatus status) {
189 case media::PIPELINE_OK:
192 case media::DECODER_ERROR_NOT_SUPPORTED:
193 result = PP_ERROR_NOTSUPPORTED;
196 result = PP_ERROR_FAILED;
200 // Calculate how many textures the shim should create.
201 uint32_t shim_texture_pool_size =
202 max_decodes_at_decoder_ + media::limits::kMaxVideoFrames;
203 main_message_loop_->PostTask(
205 base::Bind(&VideoDecoderShim::OnInitializeComplete,
208 shim_texture_pool_size));
211 void VideoDecoderShim::DecoderImpl::DoDecode() {
212 while (!pending_decodes_.empty() &&
213 num_decodes_at_decoder_ < max_decodes_at_decoder_) {
214 num_decodes_at_decoder_++;
215 const PendingDecode& decode = pending_decodes_.front();
218 base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
219 base::Unretained(this),
221 pending_decodes_.pop();
225 void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
227 media::VideoDecoder::Status status) {
228 num_decodes_at_decoder_--;
229 decode_id_ = decode_id;
233 case media::VideoDecoder::kOk:
234 case media::VideoDecoder::kAborted:
237 case media::VideoDecoder::kDecodeError:
238 result = PP_ERROR_RESOURCE_FAILED;
242 result = PP_ERROR_FAILED;
246 main_message_loop_->PostTask(
249 &VideoDecoderShim::OnDecodeComplete, shim_, result, decode_id));
254 void VideoDecoderShim::DecoderImpl::OnOutputComplete(
255 const scoped_refptr<media::VideoFrame>& frame) {
256 scoped_ptr<PendingFrame> pending_frame;
257 if (!frame->end_of_stream()) {
258 pending_frame.reset(new PendingFrame(decode_id_, frame->coded_size()));
259 // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
260 libyuv::I420ToABGR(frame->data(media::VideoFrame::kYPlane),
261 frame->stride(media::VideoFrame::kYPlane),
262 frame->data(media::VideoFrame::kUPlane),
263 frame->stride(media::VideoFrame::kUPlane),
264 frame->data(media::VideoFrame::kVPlane),
265 frame->stride(media::VideoFrame::kVPlane),
266 &pending_frame->argb_pixels.front(),
267 frame->coded_size().width() * 4,
268 frame->coded_size().width(),
269 frame->coded_size().height());
271 pending_frame.reset(new PendingFrame(decode_id_));
274 main_message_loop_->PostTask(FROM_HERE,
275 base::Bind(&VideoDecoderShim::OnOutputComplete,
277 base::Passed(&pending_frame)));
280 void VideoDecoderShim::DecoderImpl::OnResetComplete() {
281 main_message_loop_->PostTask(
282 FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_));
285 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host)
286 : state_(UNINITIALIZED),
289 RenderThreadImpl::current()->GetMediaThreadMessageLoopProxy()),
291 RenderThreadImpl::current()->SharedMainThreadContextProvider()),
292 texture_pool_size_(0),
293 num_pending_decodes_(0),
294 weak_ptr_factory_(this) {
296 DCHECK(media_message_loop_);
297 DCHECK(context_provider_);
298 decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr()));
301 VideoDecoderShim::~VideoDecoderShim() {
302 DCHECK(RenderThreadImpl::current());
303 // Delete any remaining textures.
304 TextureIdMap::iterator it = texture_id_map_.begin();
305 for (; it != texture_id_map_.end(); ++it)
306 DeleteTexture(it->second);
307 texture_id_map_.clear();
309 FlushCommandBuffer();
311 weak_ptr_factory_.InvalidateWeakPtrs();
312 // No more callbacks from the delegate will be received now.
314 // The callback now holds the only reference to the DecoderImpl, which will be
315 // deleted when Stop completes.
316 media_message_loop_->PostTask(
318 base::Bind(&VideoDecoderShim::DecoderImpl::Stop,
319 base::Owned(decoder_impl_.release())));
322 bool VideoDecoderShim::Initialize(
323 media::VideoCodecProfile profile,
324 media::VideoDecodeAccelerator::Client* client) {
325 DCHECK_EQ(client, host_);
326 DCHECK(RenderThreadImpl::current());
327 DCHECK_EQ(state_, UNINITIALIZED);
328 media::VideoCodec codec = media::kUnknownVideoCodec;
329 if (profile <= media::H264PROFILE_MAX)
330 codec = media::kCodecH264;
331 else if (profile <= media::VP8PROFILE_MAX)
332 codec = media::kCodecVP8;
333 else if (profile <= media::VP9PROFILE_MAX)
334 codec = media::kCodecVP9;
335 DCHECK_NE(codec, media::kUnknownVideoCodec);
337 media::VideoDecoderConfig config(
340 media::VideoFrame::YV12,
341 gfx::Size(32, 24), // Small sizes that won't fail.
344 NULL /* extra_data */, // TODO(bbudge) Verify this isn't needed.
345 0 /* extra_data_size */,
346 false /* decryption */);
348 media_message_loop_->PostTask(
350 base::Bind(&VideoDecoderShim::DecoderImpl::Initialize,
351 base::Unretained(decoder_impl_.get()),
353 // Return success, even though we are asynchronous, to mimic
354 // media::VideoDecodeAccelerator.
358 void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) {
359 DCHECK(RenderThreadImpl::current());
360 DCHECK_EQ(state_, DECODING);
362 // We need the address of the shared memory, so we can copy the buffer.
363 const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id());
366 media_message_loop_->PostTask(
369 &VideoDecoderShim::DecoderImpl::Decode,
370 base::Unretained(decoder_impl_.get()),
371 bitstream_buffer.id(),
372 media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
373 num_pending_decodes_++;
376 void VideoDecoderShim::AssignPictureBuffers(
377 const std::vector<media::PictureBuffer>& buffers) {
378 DCHECK(RenderThreadImpl::current());
379 DCHECK_EQ(state_, DECODING);
380 if (buffers.empty()) {
384 DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size());
385 GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
386 std::vector<uint32_t> local_texture_ids(num_textures);
387 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
388 gles2->GenTextures(num_textures, &local_texture_ids.front());
389 for (uint32_t i = 0; i < num_textures; i++) {
390 gles2->ActiveTexture(GL_TEXTURE0);
391 gles2->BindTexture(GL_TEXTURE_2D, local_texture_ids[i]);
392 gles2->ConsumeTextureCHROMIUM(GL_TEXTURE_2D,
393 pending_texture_mailboxes_[i].name);
394 // Map the plugin texture id to the local texture id.
395 uint32_t plugin_texture_id = buffers[i].texture_id();
396 texture_id_map_[plugin_texture_id] = local_texture_ids[i];
397 available_textures_.insert(plugin_texture_id);
399 pending_texture_mailboxes_.clear();
403 void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) {
404 DCHECK(RenderThreadImpl::current());
405 uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
406 if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
407 DismissTexture(texture_id);
408 } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
409 available_textures_.insert(texture_id);
416 void VideoDecoderShim::Flush() {
417 DCHECK(RenderThreadImpl::current());
418 DCHECK_EQ(state_, DECODING);
422 void VideoDecoderShim::Reset() {
423 DCHECK(RenderThreadImpl::current());
424 DCHECK_EQ(state_, DECODING);
426 media_message_loop_->PostTask(
428 base::Bind(&VideoDecoderShim::DecoderImpl::Reset,
429 base::Unretained(decoder_impl_.get())));
432 void VideoDecoderShim::Destroy() {
433 // This will be called, but our destructor does the actual work.
436 void VideoDecoderShim::OnInitializeComplete(int32_t result,
437 uint32_t texture_pool_size) {
438 DCHECK(RenderThreadImpl::current());
441 if (result == PP_OK) {
443 texture_pool_size_ = texture_pool_size;
446 host_->OnInitializeComplete(result);
449 void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) {
450 DCHECK(RenderThreadImpl::current());
453 if (result == PP_ERROR_RESOURCE_FAILED) {
454 host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
458 num_pending_decodes_--;
459 completed_decodes_.push(decode_id);
461 // If frames are being queued because we're out of textures, don't notify
462 // the host that decode has completed. This exerts "back pressure" to keep
463 // the host from sending buffers that will cause pending_frames_ to grow.
464 if (pending_frames_.empty())
465 NotifyCompletedDecodes();
468 void VideoDecoderShim::OnOutputComplete(scoped_ptr<PendingFrame> frame) {
469 DCHECK(RenderThreadImpl::current());
472 if (!frame->argb_pixels.empty()) {
473 if (texture_size_ != frame->size) {
474 // If the size has changed, all current textures must be dismissed. Add
475 // all textures to |textures_to_dismiss_| and dismiss any that aren't in
476 // use by the plugin. We will dismiss the rest as they are recycled.
477 for (TextureIdMap::const_iterator it = texture_id_map_.begin();
478 it != texture_id_map_.end();
480 textures_to_dismiss_.insert(it->second);
482 for (TextureIdSet::const_iterator it = available_textures_.begin();
483 it != available_textures_.end();
487 available_textures_.clear();
488 FlushCommandBuffer();
490 DCHECK(pending_texture_mailboxes_.empty());
491 for (uint32_t i = 0; i < texture_pool_size_; i++)
492 pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
494 host_->RequestTextures(texture_pool_size_,
497 pending_texture_mailboxes_);
498 texture_size_ = frame->size;
501 pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
506 void VideoDecoderShim::SendPictures() {
507 DCHECK(RenderThreadImpl::current());
509 while (!pending_frames_.empty() && !available_textures_.empty()) {
510 const linked_ptr<PendingFrame>& frame = pending_frames_.front();
512 TextureIdSet::iterator it = available_textures_.begin();
513 uint32_t texture_id = *it;
514 available_textures_.erase(it);
516 uint32_t local_texture_id = texture_id_map_[texture_id];
517 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
518 gles2->ActiveTexture(GL_TEXTURE0);
519 gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
520 gles2->TexImage2D(GL_TEXTURE_2D,
523 texture_size_.width(),
524 texture_size_.height(),
528 &frame->argb_pixels.front());
530 host_->PictureReady(media::Picture(texture_id, frame->decode_id));
531 pending_frames_.pop();
534 FlushCommandBuffer();
536 if (pending_frames_.empty()) {
537 // If frames aren't backing up, notify the host of any completed decodes so
538 // it can send more buffers.
539 NotifyCompletedDecodes();
541 if (state_ == FLUSHING && !num_pending_decodes_) {
543 host_->NotifyFlushDone();
548 void VideoDecoderShim::OnResetComplete() {
549 DCHECK(RenderThreadImpl::current());
552 while (!pending_frames_.empty())
553 pending_frames_.pop();
554 NotifyCompletedDecodes();
556 // Dismiss any old textures now.
557 while (!textures_to_dismiss_.empty())
558 DismissTexture(*textures_to_dismiss_.begin());
559 // Make all textures available.
560 for (TextureIdMap::const_iterator it = texture_id_map_.begin();
561 it != texture_id_map_.end();
563 available_textures_.insert(it->first);
567 host_->NotifyResetDone();
570 void VideoDecoderShim::NotifyCompletedDecodes() {
571 while (!completed_decodes_.empty()) {
572 host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front());
573 completed_decodes_.pop();
577 void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
579 textures_to_dismiss_.erase(texture_id);
580 DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
581 DeleteTexture(texture_id_map_[texture_id]);
582 texture_id_map_.erase(texture_id);
583 host_->DismissPictureBuffer(texture_id);
586 void VideoDecoderShim::DeleteTexture(uint32_t texture_id) {
587 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
588 gles2->DeleteTextures(1, &texture_id);
591 void VideoDecoderShim::FlushCommandBuffer() {
592 context_provider_->ContextGL()->Flush();
595 } // namespace content