1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/pepper/video_decoder_shim.h"
8 #include <GLES2/gl2ext.h>
9 #include <GLES2/gl2extchromium.h>
11 #include "base/bind.h"
12 #include "base/numerics/safe_conversions.h"
13 #include "content/public/renderer/render_thread.h"
14 #include "content/renderer/pepper/pepper_video_decoder_host.h"
15 #include "content/renderer/render_thread_impl.h"
16 #include "gpu/command_buffer/client/gles2_implementation.h"
17 #include "media/base/decoder_buffer.h"
18 #include "media/base/limits.h"
19 #include "media/base/video_decoder.h"
20 #include "media/filters/ffmpeg_video_decoder.h"
21 #include "media/filters/vpx_video_decoder.h"
22 #include "media/video/picture.h"
23 #include "media/video/video_decode_accelerator.h"
24 #include "ppapi/c/pp_errors.h"
25 #include "third_party/libyuv/include/libyuv.h"
26 #include "webkit/common/gpu/context_provider_web_context.h"
30 struct VideoDecoderShim::PendingDecode {
31 PendingDecode(uint32_t decode_id,
32 const scoped_refptr<media::DecoderBuffer>& buffer);
35 const uint32_t decode_id;
36 const scoped_refptr<media::DecoderBuffer> buffer;
39 VideoDecoderShim::PendingDecode::PendingDecode(
41 const scoped_refptr<media::DecoderBuffer>& buffer)
42 : decode_id(decode_id), buffer(buffer) {
45 VideoDecoderShim::PendingDecode::~PendingDecode() {
48 struct VideoDecoderShim::PendingFrame {
49 explicit PendingFrame(uint32_t decode_id);
50 PendingFrame(uint32_t decode_id,
51 const gfx::Size& coded_size,
52 const gfx::Rect& visible_rect);
55 const uint32_t decode_id;
56 const gfx::Size coded_size;
57 const gfx::Rect visible_rect;
58 std::vector<uint8_t> argb_pixels;
61 // This could be expensive to copy, so guard against that.
62 DISALLOW_COPY_AND_ASSIGN(PendingFrame);
65 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
66 : decode_id(decode_id) {
69 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id,
70 const gfx::Size& coded_size,
71 const gfx::Rect& visible_rect)
72 : decode_id(decode_id),
73 coded_size(coded_size),
74 visible_rect(visible_rect),
75 argb_pixels(coded_size.width() * coded_size.height() * 4) {
78 VideoDecoderShim::PendingFrame::~PendingFrame() {
81 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
82 // calls from the VideoDecodeShim on the main thread and sending results back.
83 // This class is constructed on the main thread, but used and destructed on the
85 class VideoDecoderShim::DecoderImpl {
87 explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
90 void Initialize(media::VideoDecoderConfig config);
91 void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
96 void OnPipelineStatus(media::PipelineStatus status);
98 void OnDecodeComplete(uint32_t decode_id, media::VideoDecoder::Status status);
99 void OnOutputComplete(const scoped_refptr<media::VideoFrame>& frame);
100 void OnResetComplete();
102 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
103 base::WeakPtr<VideoDecoderShim> shim_;
104 scoped_ptr<media::VideoDecoder> decoder_;
105 scoped_refptr<base::MessageLoopProxy> main_message_loop_;
106 // Queue of decodes waiting for the decoder.
107 typedef std::queue<PendingDecode> PendingDecodeQueue;
108 PendingDecodeQueue pending_decodes_;
109 int max_decodes_at_decoder_;
110 int num_decodes_at_decoder_;
111 // VideoDecoder returns pictures without information about the decode buffer
112 // that generated it. Save the decode_id from the last decode that completed,
113 // which is close for most decoders, which only decode one buffer at a time.
117 VideoDecoderShim::DecoderImpl::DecoderImpl(
118 const base::WeakPtr<VideoDecoderShim>& proxy)
120 main_message_loop_(base::MessageLoopProxy::current()),
121 max_decodes_at_decoder_(0),
122 num_decodes_at_decoder_(0),
126 VideoDecoderShim::DecoderImpl::~DecoderImpl() {
127 DCHECK(pending_decodes_.empty());
130 void VideoDecoderShim::DecoderImpl::Initialize(
131 media::VideoDecoderConfig config) {
133 #if !defined(MEDIA_DISABLE_LIBVPX)
134 if (config.codec() == media::kCodecVP9) {
136 new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
140 scoped_ptr<media::FFmpegVideoDecoder> ffmpeg_video_decoder(
141 new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
142 ffmpeg_video_decoder->set_decode_nalus(true);
143 decoder_ = ffmpeg_video_decoder.Pass();
145 max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests();
146 // We can use base::Unretained() safely in decoder callbacks because
147 // |decoder_| is owned by DecoderImpl. During Stop(), the |decoder_| will be
148 // destroyed and all outstanding callbacks will be fired.
149 decoder_->Initialize(
151 true /* low_delay */,
152 base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus,
153 base::Unretained(this)),
154 base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete,
155 base::Unretained(this)));
158 void VideoDecoderShim::DecoderImpl::Decode(
160 scoped_refptr<media::DecoderBuffer> buffer) {
162 pending_decodes_.push(PendingDecode(decode_id, buffer));
166 void VideoDecoderShim::DecoderImpl::Reset() {
168 // Abort all pending decodes.
169 while (!pending_decodes_.empty()) {
170 const PendingDecode& decode = pending_decodes_.front();
171 scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
172 main_message_loop_->PostTask(FROM_HERE,
173 base::Bind(&VideoDecoderShim::OnDecodeComplete,
175 media::VideoDecoder::kAborted,
177 pending_decodes_.pop();
179 decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete,
180 base::Unretained(this)));
183 void VideoDecoderShim::DecoderImpl::Stop() {
185 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
187 while (!pending_decodes_.empty())
188 pending_decodes_.pop();
190 // This instance is deleted once we exit this scope.
193 void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
194 media::PipelineStatus status) {
197 case media::PIPELINE_OK:
200 case media::DECODER_ERROR_NOT_SUPPORTED:
201 result = PP_ERROR_NOTSUPPORTED;
204 result = PP_ERROR_FAILED;
208 // Calculate how many textures the shim should create.
209 uint32_t shim_texture_pool_size =
210 max_decodes_at_decoder_ + media::limits::kMaxVideoFrames;
211 main_message_loop_->PostTask(
213 base::Bind(&VideoDecoderShim::OnInitializeComplete,
216 shim_texture_pool_size));
219 void VideoDecoderShim::DecoderImpl::DoDecode() {
220 while (!pending_decodes_.empty() &&
221 num_decodes_at_decoder_ < max_decodes_at_decoder_) {
222 num_decodes_at_decoder_++;
223 const PendingDecode& decode = pending_decodes_.front();
226 base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
227 base::Unretained(this),
229 pending_decodes_.pop();
233 void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
235 media::VideoDecoder::Status status) {
236 num_decodes_at_decoder_--;
237 decode_id_ = decode_id;
241 case media::VideoDecoder::kOk:
242 case media::VideoDecoder::kAborted:
245 case media::VideoDecoder::kDecodeError:
246 result = PP_ERROR_RESOURCE_FAILED;
250 result = PP_ERROR_FAILED;
254 main_message_loop_->PostTask(
257 &VideoDecoderShim::OnDecodeComplete, shim_, result, decode_id));
262 void VideoDecoderShim::DecoderImpl::OnOutputComplete(
263 const scoped_refptr<media::VideoFrame>& frame) {
264 scoped_ptr<PendingFrame> pending_frame;
265 if (!frame->end_of_stream()) {
266 pending_frame.reset(new PendingFrame(
267 decode_id_, frame->coded_size(), frame->visible_rect()));
268 // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
269 libyuv::I420ToABGR(frame->data(media::VideoFrame::kYPlane),
270 frame->stride(media::VideoFrame::kYPlane),
271 frame->data(media::VideoFrame::kUPlane),
272 frame->stride(media::VideoFrame::kUPlane),
273 frame->data(media::VideoFrame::kVPlane),
274 frame->stride(media::VideoFrame::kVPlane),
275 &pending_frame->argb_pixels.front(),
276 frame->coded_size().width() * 4,
277 frame->coded_size().width(),
278 frame->coded_size().height());
280 pending_frame.reset(new PendingFrame(decode_id_));
283 main_message_loop_->PostTask(FROM_HERE,
284 base::Bind(&VideoDecoderShim::OnOutputComplete,
286 base::Passed(&pending_frame)));
289 void VideoDecoderShim::DecoderImpl::OnResetComplete() {
290 main_message_loop_->PostTask(
291 FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_));
294 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host)
295 : state_(UNINITIALIZED),
298 RenderThreadImpl::current()->GetMediaThreadMessageLoopProxy()),
300 RenderThreadImpl::current()->SharedMainThreadContextProvider()),
301 texture_pool_size_(0),
302 num_pending_decodes_(0),
303 weak_ptr_factory_(this) {
305 DCHECK(media_message_loop_);
306 DCHECK(context_provider_);
307 decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr()));
310 VideoDecoderShim::~VideoDecoderShim() {
311 DCHECK(RenderThreadImpl::current());
312 // Delete any remaining textures.
313 TextureIdMap::iterator it = texture_id_map_.begin();
314 for (; it != texture_id_map_.end(); ++it)
315 DeleteTexture(it->second);
316 texture_id_map_.clear();
318 FlushCommandBuffer();
320 weak_ptr_factory_.InvalidateWeakPtrs();
321 // No more callbacks from the delegate will be received now.
323 // The callback now holds the only reference to the DecoderImpl, which will be
324 // deleted when Stop completes.
325 media_message_loop_->PostTask(
327 base::Bind(&VideoDecoderShim::DecoderImpl::Stop,
328 base::Owned(decoder_impl_.release())));
331 bool VideoDecoderShim::Initialize(
332 media::VideoCodecProfile profile,
333 media::VideoDecodeAccelerator::Client* client) {
334 DCHECK_EQ(client, host_);
335 DCHECK(RenderThreadImpl::current());
336 DCHECK_EQ(state_, UNINITIALIZED);
337 media::VideoCodec codec = media::kUnknownVideoCodec;
338 if (profile <= media::H264PROFILE_MAX)
339 codec = media::kCodecH264;
340 else if (profile <= media::VP8PROFILE_MAX)
341 codec = media::kCodecVP8;
342 else if (profile <= media::VP9PROFILE_MAX)
343 codec = media::kCodecVP9;
344 DCHECK_NE(codec, media::kUnknownVideoCodec);
346 media::VideoDecoderConfig config(
349 media::VideoFrame::YV12,
350 gfx::Size(32, 24), // Small sizes that won't fail.
353 NULL /* extra_data */, // TODO(bbudge) Verify this isn't needed.
354 0 /* extra_data_size */,
355 false /* decryption */);
357 media_message_loop_->PostTask(
359 base::Bind(&VideoDecoderShim::DecoderImpl::Initialize,
360 base::Unretained(decoder_impl_.get()),
362 // Return success, even though we are asynchronous, to mimic
363 // media::VideoDecodeAccelerator.
367 void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) {
368 DCHECK(RenderThreadImpl::current());
369 DCHECK_EQ(state_, DECODING);
371 // We need the address of the shared memory, so we can copy the buffer.
372 const uint8_t* buffer = host_->DecodeIdToAddress(bitstream_buffer.id());
375 media_message_loop_->PostTask(
378 &VideoDecoderShim::DecoderImpl::Decode,
379 base::Unretained(decoder_impl_.get()),
380 bitstream_buffer.id(),
381 media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
382 num_pending_decodes_++;
385 void VideoDecoderShim::AssignPictureBuffers(
386 const std::vector<media::PictureBuffer>& buffers) {
387 DCHECK(RenderThreadImpl::current());
388 DCHECK_EQ(state_, DECODING);
389 if (buffers.empty()) {
393 DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size());
394 GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
395 std::vector<uint32_t> local_texture_ids(num_textures);
396 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
397 for (uint32_t i = 0; i < num_textures; i++) {
398 local_texture_ids[i] = gles2->CreateAndConsumeTextureCHROMIUM(
399 GL_TEXTURE_2D, pending_texture_mailboxes_[i].name);
400 // Map the plugin texture id to the local texture id.
401 uint32_t plugin_texture_id = buffers[i].texture_id();
402 texture_id_map_[plugin_texture_id] = local_texture_ids[i];
403 available_textures_.insert(plugin_texture_id);
405 pending_texture_mailboxes_.clear();
409 void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) {
410 DCHECK(RenderThreadImpl::current());
411 uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
412 if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
413 DismissTexture(texture_id);
414 } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
415 available_textures_.insert(texture_id);
422 void VideoDecoderShim::Flush() {
423 DCHECK(RenderThreadImpl::current());
424 DCHECK_EQ(state_, DECODING);
428 void VideoDecoderShim::Reset() {
429 DCHECK(RenderThreadImpl::current());
430 DCHECK_EQ(state_, DECODING);
432 media_message_loop_->PostTask(
434 base::Bind(&VideoDecoderShim::DecoderImpl::Reset,
435 base::Unretained(decoder_impl_.get())));
438 void VideoDecoderShim::Destroy() {
439 // This will be called, but our destructor does the actual work.
442 void VideoDecoderShim::OnInitializeComplete(int32_t result,
443 uint32_t texture_pool_size) {
444 DCHECK(RenderThreadImpl::current());
447 if (result == PP_OK) {
449 texture_pool_size_ = texture_pool_size;
452 host_->OnInitializeComplete(result);
455 void VideoDecoderShim::OnDecodeComplete(int32_t result, uint32_t decode_id) {
456 DCHECK(RenderThreadImpl::current());
459 if (result == PP_ERROR_RESOURCE_FAILED) {
460 host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
464 num_pending_decodes_--;
465 completed_decodes_.push(decode_id);
467 // If frames are being queued because we're out of textures, don't notify
468 // the host that decode has completed. This exerts "back pressure" to keep
469 // the host from sending buffers that will cause pending_frames_ to grow.
470 if (pending_frames_.empty())
471 NotifyCompletedDecodes();
474 void VideoDecoderShim::OnOutputComplete(scoped_ptr<PendingFrame> frame) {
475 DCHECK(RenderThreadImpl::current());
478 if (!frame->argb_pixels.empty()) {
479 if (texture_size_ != frame->coded_size) {
480 // If the size has changed, all current textures must be dismissed. Add
481 // all textures to |textures_to_dismiss_| and dismiss any that aren't in
482 // use by the plugin. We will dismiss the rest as they are recycled.
483 for (TextureIdMap::const_iterator it = texture_id_map_.begin();
484 it != texture_id_map_.end();
486 textures_to_dismiss_.insert(it->second);
488 for (TextureIdSet::const_iterator it = available_textures_.begin();
489 it != available_textures_.end();
493 available_textures_.clear();
494 FlushCommandBuffer();
496 DCHECK(pending_texture_mailboxes_.empty());
497 for (uint32_t i = 0; i < texture_pool_size_; i++)
498 pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
500 host_->RequestTextures(texture_pool_size_,
503 pending_texture_mailboxes_);
504 texture_size_ = frame->coded_size;
507 pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
512 void VideoDecoderShim::SendPictures() {
513 DCHECK(RenderThreadImpl::current());
515 while (!pending_frames_.empty() && !available_textures_.empty()) {
516 const linked_ptr<PendingFrame>& frame = pending_frames_.front();
518 TextureIdSet::iterator it = available_textures_.begin();
519 uint32_t texture_id = *it;
520 available_textures_.erase(it);
522 uint32_t local_texture_id = texture_id_map_[texture_id];
523 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
524 gles2->ActiveTexture(GL_TEXTURE0);
525 gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
526 gles2->TexImage2D(GL_TEXTURE_2D,
529 texture_size_.width(),
530 texture_size_.height(),
534 &frame->argb_pixels.front());
537 media::Picture(texture_id, frame->decode_id, frame->visible_rect));
538 pending_frames_.pop();
541 FlushCommandBuffer();
543 if (pending_frames_.empty()) {
544 // If frames aren't backing up, notify the host of any completed decodes so
545 // it can send more buffers.
546 NotifyCompletedDecodes();
548 if (state_ == FLUSHING && !num_pending_decodes_) {
550 host_->NotifyFlushDone();
555 void VideoDecoderShim::OnResetComplete() {
556 DCHECK(RenderThreadImpl::current());
559 while (!pending_frames_.empty())
560 pending_frames_.pop();
561 NotifyCompletedDecodes();
563 // Dismiss any old textures now.
564 while (!textures_to_dismiss_.empty())
565 DismissTexture(*textures_to_dismiss_.begin());
568 host_->NotifyResetDone();
571 void VideoDecoderShim::NotifyCompletedDecodes() {
572 while (!completed_decodes_.empty()) {
573 host_->NotifyEndOfBitstreamBuffer(completed_decodes_.front());
574 completed_decodes_.pop();
578 void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
580 textures_to_dismiss_.erase(texture_id);
581 DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
582 DeleteTexture(texture_id_map_[texture_id]);
583 texture_id_map_.erase(texture_id);
584 host_->DismissPictureBuffer(texture_id);
587 void VideoDecoderShim::DeleteTexture(uint32_t texture_id) {
588 gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
589 gles2->DeleteTextures(1, &texture_id);
592 void VideoDecoderShim::FlushCommandBuffer() {
593 context_provider_->ContextGL()->Flush();
596 } // namespace content