2 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
24 #include "gstnvvp8dec.h"
25 #include "gstcudautils.h"
26 #include "gstnvdecoder.h"
30 GST_DEBUG_CATEGORY_STATIC (gst_nv_vp8_dec_debug);
31 #define GST_CAT_DEFAULT gst_nv_vp8_dec_debug
33 /* reference list 4 + 2 margin */
34 #define NUM_OUTPUT_VIEW 6
40 GstVideoCodecState *output_state;
42 GstCudaContext *context;
43 GstNvDecoder *decoder;
44 CUVIDPICPARAMS params;
49 struct _GstNvVp8DecClass
51 GstVp8DecoderClass parent_class;
55 #define gst_nv_vp8_dec_parent_class parent_class
62 G_DEFINE_TYPE (GstNvVp8Dec, gst_nv_vp8_dec, GST_TYPE_VP8_DECODER);
64 static void gst_nv_vp8_dec_set_context (GstElement * element,
65 GstContext * context);
66 static gboolean gst_nv_vp8_dec_open (GstVideoDecoder * decoder);
67 static gboolean gst_nv_vp8_dec_close (GstVideoDecoder * decoder);
68 static gboolean gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder);
69 static gboolean gst_nv_vp8_dec_decide_allocation (GstVideoDecoder *
70 decoder, GstQuery * query);
71 static gboolean gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder,
75 static GstFlowReturn gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
76 const GstVp8FrameHdr * frame_hdr);
77 static GstFlowReturn gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
78 GstVideoCodecFrame * frame, GstVp8Picture * picture);
79 static GstFlowReturn gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
80 GstVp8Picture * picture, GstVp8Parser * parser);
81 static GstFlowReturn gst_nv_vp8_dec_output_picture (GstVp8Decoder *
82 decoder, GstVideoCodecFrame * frame, GstVp8Picture * picture);
85 gst_nv_vp8_dec_class_init (GstNvVp8DecClass * klass)
87 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
88 GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
89 GstVp8DecoderClass *vp8decoder_class = GST_VP8_DECODER_CLASS (klass);
91 element_class->set_context = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_set_context);
93 decoder_class->open = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_open);
94 decoder_class->close = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_close);
95 decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_negotiate);
96 decoder_class->decide_allocation =
97 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decide_allocation);
98 decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_src_query);
100 vp8decoder_class->new_sequence =
101 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_sequence);
102 vp8decoder_class->new_picture =
103 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_picture);
104 vp8decoder_class->decode_picture =
105 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decode_picture);
106 vp8decoder_class->output_picture =
107 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_output_picture);
109 GST_DEBUG_CATEGORY_INIT (gst_nv_vp8_dec_debug,
110 "nvvp8dec", 0, "NVIDIA VP8 Decoder");
112 gst_type_mark_as_plugin_api (GST_TYPE_NV_VP8_DEC, 0);
116 gst_nv_vp8_dec_init (GstNvVp8Dec * self)
121 gst_nv_vp8_dec_set_context (GstElement * element, GstContext * context)
123 GstNvVp8Dec *self = GST_NV_VP8_DEC (element);
124 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
126 GST_DEBUG_OBJECT (self, "set context %s",
127 gst_context_get_context_type (context));
129 if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
135 gst_nv_decoder_handle_set_context (self->decoder, element, context);
138 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
142 gst_nv_vp8_dec_open (GstVideoDecoder * decoder)
144 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
145 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
147 if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
148 klass->cuda_device_id, &self->context)) {
149 GST_ERROR_OBJECT (self, "Required element data is unavailable");
153 self->decoder = gst_nv_decoder_new (self->context);
154 if (!self->decoder) {
155 GST_ERROR_OBJECT (self, "Failed to create decoder object");
156 gst_clear_object (&self->context);
165 gst_nv_vp8_dec_close (GstVideoDecoder * decoder)
167 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
169 g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
170 gst_clear_object (&self->decoder);
171 gst_clear_object (&self->context);
177 gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder)
179 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
180 GstVp8Decoder *vp8dec = GST_VP8_DECODER (decoder);
182 GST_DEBUG_OBJECT (self, "negotiate");
184 gst_nv_decoder_negotiate (self->decoder, decoder, vp8dec->input_state,
185 &self->output_state);
187 /* TODO: add support D3D11 memory */
189 return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
193 gst_nv_vp8_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
195 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
197 if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
198 GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
202 return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
207 gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
209 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
211 switch (GST_QUERY_TYPE (query)) {
212 case GST_QUERY_CONTEXT:
213 if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
216 } else if (self->decoder &&
217 gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
225 return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
229 gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
230 const GstVp8FrameHdr * frame_hdr)
232 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
233 gboolean modified = FALSE;
235 GST_LOG_OBJECT (self, "new sequence");
237 if (self->width != frame_hdr->width || self->height != frame_hdr->height) {
239 GST_INFO_OBJECT (self, "resolution changed %dx%d -> %dx%d",
240 self->width, self->height, frame_hdr->width, frame_hdr->height);
243 self->width = frame_hdr->width;
244 self->height = frame_hdr->height;
249 if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
252 gst_video_info_set_format (&info,
253 GST_VIDEO_FORMAT_NV12, self->width, self->height);
255 if (!gst_nv_decoder_configure (self->decoder,
256 cudaVideoCodec_VP8, &info, self->width, self->height,
258 GST_ERROR_OBJECT (self, "Failed to configure decoder");
259 return GST_FLOW_NOT_NEGOTIATED;
262 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
263 GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
264 return GST_FLOW_NOT_NEGOTIATED;
267 memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
269 self->params.PicWidthInMbs = GST_ROUND_UP_16 (self->width) >> 4;
270 self->params.FrameHeightInMbs = GST_ROUND_UP_16 (self->height) >> 4;
272 self->params.CodecSpecific.vp8.width = self->width;
273 self->params.CodecSpecific.vp8.height = self->height;
280 gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
281 GstVideoCodecFrame * frame, GstVp8Picture * picture)
283 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
284 GstNvDecoderFrame *nv_frame;
286 nv_frame = gst_nv_decoder_new_frame (self->decoder);
288 GST_ERROR_OBJECT (self, "No available decoder frame");
289 return GST_FLOW_ERROR;
292 GST_LOG_OBJECT (self,
293 "New decoder frame %p (index %d)", nv_frame, nv_frame->index);
295 gst_vp8_picture_set_user_data (picture,
296 nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
301 static GstNvDecoderFrame *
302 gst_nv_vp8_dec_get_decoder_frame_from_picture (GstNvVp8Dec * self,
303 GstVp8Picture * picture)
305 GstNvDecoderFrame *frame;
307 frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
310 GST_DEBUG_OBJECT (self, "current picture does not have decoder frame");
316 gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
317 GstVp8Picture * picture, GstVp8Parser * parser)
319 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
320 GstVp8FrameHdr *frame_hdr = &picture->frame_hdr;
321 GstNvDecoderFrame *frame;
322 GstNvDecoderFrame *other_frame;
325 GST_LOG_OBJECT (self, "Decode picture, size %" G_GSIZE_FORMAT, picture->size);
327 frame = gst_nv_vp8_dec_get_decoder_frame_from_picture (self, picture);
329 GST_ERROR_OBJECT (self, "Decoder frame is unavailable");
330 return GST_FLOW_ERROR;
333 self->params.nBitstreamDataLen = picture->size;
334 self->params.pBitstreamData = picture->data;
335 self->params.nNumSlices = 1;
336 self->params.pSliceDataOffsets = &offset;
338 self->params.CurrPicIdx = frame->index;
340 self->params.CodecSpecific.vp8.first_partition_size =
341 frame_hdr->first_part_size;
343 if (decoder->alt_ref_picture) {
345 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
346 decoder->alt_ref_picture);
348 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for AltRef");
349 return GST_FLOW_ERROR;
352 self->params.CodecSpecific.vp8.AltRefIdx = other_frame->index;
354 self->params.CodecSpecific.vp8.AltRefIdx = 0xff;
357 if (decoder->golden_ref_picture) {
359 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
360 decoder->golden_ref_picture);
362 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for GoldenRef");
363 return GST_FLOW_ERROR;
366 self->params.CodecSpecific.vp8.GoldenRefIdx = other_frame->index;
368 self->params.CodecSpecific.vp8.GoldenRefIdx = 0xff;
371 if (decoder->last_picture) {
373 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
374 decoder->last_picture);
376 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for LastRef");
377 return GST_FLOW_ERROR;
380 self->params.CodecSpecific.vp8.LastRefIdx = other_frame->index;
382 self->params.CodecSpecific.vp8.LastRefIdx = 0xff;
385 self->params.CodecSpecific.vp8.vp8_frame_tag.frame_type =
386 frame_hdr->key_frame ? 0 : 1;
387 self->params.CodecSpecific.vp8.vp8_frame_tag.version = frame_hdr->version;
388 self->params.CodecSpecific.vp8.vp8_frame_tag.show_frame =
389 frame_hdr->show_frame;
390 self->params.CodecSpecific.vp8.vp8_frame_tag.update_mb_segmentation_data =
391 parser->segmentation.segmentation_enabled ?
392 parser->segmentation.update_segment_feature_data : 0;
394 if (!gst_nv_decoder_decode_picture (self->decoder, &self->params))
395 return GST_FLOW_ERROR;
401 gst_nv_vp8_dec_output_picture (GstVp8Decoder * decoder,
402 GstVideoCodecFrame * frame, GstVp8Picture * picture)
404 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
405 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
406 GstNvDecoderFrame *decoder_frame;
408 GST_LOG_OBJECT (self, "Outputting picture %p", picture);
410 decoder_frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
411 if (!decoder_frame) {
412 GST_ERROR_OBJECT (self, "No decoder frame in picture %p", picture);
416 if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
417 &frame->output_buffer)) {
418 GST_ERROR_OBJECT (self, "Failed to handle output picture");
422 gst_vp8_picture_unref (picture);
424 return gst_video_decoder_finish_frame (vdec, frame);
427 gst_video_decoder_drop_frame (vdec, frame);
428 gst_vp8_picture_unref (picture);
430 return GST_FLOW_ERROR;
437 guint cuda_device_id;
439 } GstNvVp8DecClassData;
442 gst_nv_vp8_dec_subclass_init (gpointer klass, GstNvVp8DecClassData * cdata)
444 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
445 GstNvVp8DecClass *nvdec_class = (GstNvVp8DecClass *) (klass);
448 if (cdata->is_default) {
449 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder");
451 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder with device %d",
452 cdata->cuda_device_id);
455 gst_element_class_set_metadata (element_class, long_name,
456 "Codec/Decoder/Video/Hardware",
457 "NVIDIA VP8 video decoder", "Seungha Yang <seungha@centricular.com>");
460 gst_element_class_add_pad_template (element_class,
461 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
463 gst_element_class_add_pad_template (element_class,
464 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
467 nvdec_class->cuda_device_id = cdata->cuda_device_id;
469 gst_caps_unref (cdata->sink_caps);
470 gst_caps_unref (cdata->src_caps);
475 gst_nv_vp8_dec_register (GstPlugin * plugin, guint device_id, guint rank,
476 GstCaps * sink_caps, GstCaps * src_caps, gboolean is_primary)
478 GTypeQuery type_query;
479 GTypeInfo type_info = { 0, };
483 GstNvVp8DecClassData *cdata;
484 gboolean is_default = TRUE;
487 * element-nvvp8sldec:
492 cdata = g_new0 (GstNvVp8DecClassData, 1);
493 cdata->sink_caps = gst_caps_ref (sink_caps);
494 cdata->src_caps = gst_caps_ref (src_caps);
495 cdata->cuda_device_id = device_id;
497 g_type_query (GST_TYPE_NV_VP8_DEC, &type_query);
498 memset (&type_info, 0, sizeof (type_info));
499 type_info.class_size = type_query.class_size;
500 type_info.instance_size = type_query.instance_size;
501 type_info.class_init = (GClassInitFunc) gst_nv_vp8_dec_subclass_init;
502 type_info.class_data = cdata;
505 type_name = g_strdup ("GstNvVP8StatelessPrimaryDec");
506 feature_name = g_strdup ("nvvp8dec");
508 type_name = g_strdup ("GstNvVP8StatelessDec");
509 feature_name = g_strdup ("nvvp8sldec");
512 if (g_type_from_name (type_name) != 0) {
514 g_free (feature_name);
517 g_strdup_printf ("GstNvVP8StatelessPrimaryDevice%dDec", device_id);
518 feature_name = g_strdup_printf ("nvvp8device%ddec", device_id);
520 type_name = g_strdup_printf ("GstNvVP8StatelessDevice%dDec", device_id);
521 feature_name = g_strdup_printf ("nvvp8sldevice%ddec", device_id);
527 cdata->is_default = is_default;
528 subtype = g_type_register_static (GST_TYPE_NV_VP8_DEC,
529 type_name, &type_info, 0);
531 /* make lower rank than default device */
532 if (rank > 0 && !is_default)
535 if (!gst_element_register (plugin, feature_name, rank, subtype))
536 GST_WARNING ("Failed to register plugin '%s'", type_name);
539 g_free (feature_name);