2 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
24 #include <gst/cuda/gstcudautils.h>
26 #include "gstnvvp8dec.h"
27 #include "gstnvdecoder.h"
31 GST_DEBUG_CATEGORY_STATIC (gst_nv_vp8_dec_debug);
32 #define GST_CAT_DEFAULT gst_nv_vp8_dec_debug
38 GstVideoCodecState *output_state;
40 GstCudaContext *context;
41 GstNvDecoder *decoder;
42 CUVIDPICPARAMS params;
47 struct _GstNvVp8DecClass
49 GstVp8DecoderClass parent_class;
53 #define gst_nv_vp8_dec_parent_class parent_class
60 G_DEFINE_TYPE (GstNvVp8Dec, gst_nv_vp8_dec, GST_TYPE_VP8_DECODER);
62 static void gst_nv_vp8_dec_set_context (GstElement * element,
63 GstContext * context);
64 static gboolean gst_nv_vp8_dec_open (GstVideoDecoder * decoder);
65 static gboolean gst_nv_vp8_dec_close (GstVideoDecoder * decoder);
66 static gboolean gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder);
67 static gboolean gst_nv_vp8_dec_decide_allocation (GstVideoDecoder *
68 decoder, GstQuery * query);
69 static gboolean gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder,
73 static GstFlowReturn gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
74 const GstVp8FrameHdr * frame_hdr, gint max_dpb_size);
75 static GstFlowReturn gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
76 GstVideoCodecFrame * frame, GstVp8Picture * picture);
77 static GstFlowReturn gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
78 GstVp8Picture * picture, GstVp8Parser * parser);
79 static GstFlowReturn gst_nv_vp8_dec_output_picture (GstVp8Decoder *
80 decoder, GstVideoCodecFrame * frame, GstVp8Picture * picture);
81 static guint gst_nv_vp8_dec_get_preferred_output_delay (GstVp8Decoder * decoder,
85 gst_nv_vp8_dec_class_init (GstNvVp8DecClass * klass)
87 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
88 GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
89 GstVp8DecoderClass *vp8decoder_class = GST_VP8_DECODER_CLASS (klass);
91 element_class->set_context = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_set_context);
93 decoder_class->open = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_open);
94 decoder_class->close = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_close);
95 decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_negotiate);
96 decoder_class->decide_allocation =
97 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decide_allocation);
98 decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_src_query);
100 vp8decoder_class->new_sequence =
101 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_sequence);
102 vp8decoder_class->new_picture =
103 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_picture);
104 vp8decoder_class->decode_picture =
105 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decode_picture);
106 vp8decoder_class->output_picture =
107 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_output_picture);
108 vp8decoder_class->get_preferred_output_delay =
109 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_get_preferred_output_delay);
111 GST_DEBUG_CATEGORY_INIT (gst_nv_vp8_dec_debug,
112 "nvvp8dec", 0, "NVIDIA VP8 Decoder");
114 gst_type_mark_as_plugin_api (GST_TYPE_NV_VP8_DEC, 0);
118 gst_nv_vp8_dec_init (GstNvVp8Dec * self)
123 gst_nv_vp8_dec_set_context (GstElement * element, GstContext * context)
125 GstNvVp8Dec *self = GST_NV_VP8_DEC (element);
126 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
128 GST_DEBUG_OBJECT (self, "set context %s",
129 gst_context_get_context_type (context));
131 if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
137 gst_nv_decoder_handle_set_context (self->decoder, element, context);
140 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
144 gst_nv_vp8_dec_open (GstVideoDecoder * decoder)
146 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
147 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
149 if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
150 klass->cuda_device_id, &self->context)) {
151 GST_ERROR_OBJECT (self, "Required element data is unavailable");
155 self->decoder = gst_nv_decoder_new (self->context);
156 if (!self->decoder) {
157 GST_ERROR_OBJECT (self, "Failed to create decoder object");
158 gst_clear_object (&self->context);
167 gst_nv_vp8_dec_close (GstVideoDecoder * decoder)
169 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
171 g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
172 gst_clear_object (&self->decoder);
173 gst_clear_object (&self->context);
179 gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder)
181 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
182 GstVp8Decoder *vp8dec = GST_VP8_DECODER (decoder);
184 GST_DEBUG_OBJECT (self, "negotiate");
186 gst_nv_decoder_negotiate (self->decoder, decoder, vp8dec->input_state,
187 &self->output_state);
189 /* TODO: add support D3D11 memory */
191 return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
195 gst_nv_vp8_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
197 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
199 if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
200 GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
204 return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
209 gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
211 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
213 switch (GST_QUERY_TYPE (query)) {
214 case GST_QUERY_CONTEXT:
215 if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
218 } else if (self->decoder &&
219 gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
227 return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
231 gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
232 const GstVp8FrameHdr * frame_hdr, gint max_dpb_size)
234 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
235 gboolean modified = FALSE;
237 GST_LOG_OBJECT (self, "new sequence");
239 if (self->width != frame_hdr->width || self->height != frame_hdr->height) {
241 GST_INFO_OBJECT (self, "resolution changed %dx%d -> %dx%d",
242 self->width, self->height, frame_hdr->width, frame_hdr->height);
245 self->width = frame_hdr->width;
246 self->height = frame_hdr->height;
251 if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
254 gst_video_info_set_format (&info,
255 GST_VIDEO_FORMAT_NV12, self->width, self->height);
257 if (!gst_nv_decoder_configure (self->decoder,
258 cudaVideoCodec_VP8, &info, self->width, self->height, 8,
259 max_dpb_size, FALSE)) {
260 GST_ERROR_OBJECT (self, "Failed to configure decoder");
261 return GST_FLOW_NOT_NEGOTIATED;
264 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
265 GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
266 return GST_FLOW_NOT_NEGOTIATED;
269 memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
271 self->params.PicWidthInMbs = GST_ROUND_UP_16 (self->width) >> 4;
272 self->params.FrameHeightInMbs = GST_ROUND_UP_16 (self->height) >> 4;
274 self->params.CodecSpecific.vp8.width = self->width;
275 self->params.CodecSpecific.vp8.height = self->height;
282 gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
283 GstVideoCodecFrame * frame, GstVp8Picture * picture)
285 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
286 GstNvDecoderFrame *nv_frame;
288 nv_frame = gst_nv_decoder_new_frame (self->decoder);
290 GST_ERROR_OBJECT (self, "No available decoder frame");
291 return GST_FLOW_ERROR;
294 GST_LOG_OBJECT (self,
295 "New decoder frame %p (index %d)", nv_frame, nv_frame->index);
297 gst_vp8_picture_set_user_data (picture,
298 nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
303 static GstNvDecoderFrame *
304 gst_nv_vp8_dec_get_decoder_frame_from_picture (GstNvVp8Dec * self,
305 GstVp8Picture * picture)
307 GstNvDecoderFrame *frame;
309 frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
312 GST_DEBUG_OBJECT (self, "current picture does not have decoder frame");
318 gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
319 GstVp8Picture * picture, GstVp8Parser * parser)
321 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
322 GstVp8FrameHdr *frame_hdr = &picture->frame_hdr;
323 GstNvDecoderFrame *frame;
324 GstNvDecoderFrame *other_frame;
327 GST_LOG_OBJECT (self, "Decode picture, size %" G_GSIZE_FORMAT, picture->size);
329 frame = gst_nv_vp8_dec_get_decoder_frame_from_picture (self, picture);
331 GST_ERROR_OBJECT (self, "Decoder frame is unavailable");
332 return GST_FLOW_ERROR;
335 self->params.nBitstreamDataLen = picture->size;
336 self->params.pBitstreamData = picture->data;
337 self->params.nNumSlices = 1;
338 self->params.pSliceDataOffsets = &offset;
340 self->params.CurrPicIdx = frame->index;
342 self->params.CodecSpecific.vp8.first_partition_size =
343 frame_hdr->first_part_size;
345 if (decoder->alt_ref_picture) {
347 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
348 decoder->alt_ref_picture);
350 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for AltRef");
351 return GST_FLOW_ERROR;
354 self->params.CodecSpecific.vp8.AltRefIdx = other_frame->index;
356 self->params.CodecSpecific.vp8.AltRefIdx = 0xff;
359 if (decoder->golden_ref_picture) {
361 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
362 decoder->golden_ref_picture);
364 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for GoldenRef");
365 return GST_FLOW_ERROR;
368 self->params.CodecSpecific.vp8.GoldenRefIdx = other_frame->index;
370 self->params.CodecSpecific.vp8.GoldenRefIdx = 0xff;
373 if (decoder->last_picture) {
375 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
376 decoder->last_picture);
378 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for LastRef");
379 return GST_FLOW_ERROR;
382 self->params.CodecSpecific.vp8.LastRefIdx = other_frame->index;
384 self->params.CodecSpecific.vp8.LastRefIdx = 0xff;
387 self->params.CodecSpecific.vp8.vp8_frame_tag.frame_type =
388 frame_hdr->key_frame ? 0 : 1;
389 self->params.CodecSpecific.vp8.vp8_frame_tag.version = frame_hdr->version;
390 self->params.CodecSpecific.vp8.vp8_frame_tag.show_frame =
391 frame_hdr->show_frame;
392 self->params.CodecSpecific.vp8.vp8_frame_tag.update_mb_segmentation_data =
393 parser->segmentation.segmentation_enabled ?
394 parser->segmentation.update_segment_feature_data : 0;
396 if (!gst_nv_decoder_decode_picture (self->decoder, &self->params))
397 return GST_FLOW_ERROR;
403 gst_nv_vp8_dec_output_picture (GstVp8Decoder * decoder,
404 GstVideoCodecFrame * frame, GstVp8Picture * picture)
406 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
407 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
408 GstNvDecoderFrame *decoder_frame;
410 GST_LOG_OBJECT (self, "Outputting picture %p", picture);
412 decoder_frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
413 if (!decoder_frame) {
414 GST_ERROR_OBJECT (self, "No decoder frame in picture %p", picture);
418 if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
419 &frame->output_buffer)) {
420 GST_ERROR_OBJECT (self, "Failed to handle output picture");
424 gst_vp8_picture_unref (picture);
426 return gst_video_decoder_finish_frame (vdec, frame);
429 gst_video_decoder_drop_frame (vdec, frame);
430 gst_vp8_picture_unref (picture);
432 return GST_FLOW_ERROR;
436 gst_nv_vp8_dec_get_preferred_output_delay (GstVp8Decoder * decoder,
439 /* Prefer to zero latency for live pipeline */
443 /* NVCODEC SDK uses 4 frame delay for better throughput performance */
451 guint cuda_device_id;
453 } GstNvVp8DecClassData;
456 gst_nv_vp8_dec_subclass_init (gpointer klass, GstNvVp8DecClassData * cdata)
458 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
459 GstNvVp8DecClass *nvdec_class = (GstNvVp8DecClass *) (klass);
462 if (cdata->is_default) {
463 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder");
465 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder with device %d",
466 cdata->cuda_device_id);
469 gst_element_class_set_metadata (element_class, long_name,
470 "Codec/Decoder/Video/Hardware",
471 "NVIDIA VP8 video decoder", "Seungha Yang <seungha@centricular.com>");
474 gst_element_class_add_pad_template (element_class,
475 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
477 gst_element_class_add_pad_template (element_class,
478 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
481 nvdec_class->cuda_device_id = cdata->cuda_device_id;
483 gst_caps_unref (cdata->sink_caps);
484 gst_caps_unref (cdata->src_caps);
489 gst_nv_vp8_dec_register (GstPlugin * plugin, guint device_id, guint rank,
490 GstCaps * sink_caps, GstCaps * src_caps, gboolean is_primary)
492 GTypeQuery type_query;
493 GTypeInfo type_info = { 0, };
497 GstNvVp8DecClassData *cdata;
498 gboolean is_default = TRUE;
501 * element-nvvp8sldec:
506 cdata = g_new0 (GstNvVp8DecClassData, 1);
507 cdata->sink_caps = gst_caps_ref (sink_caps);
508 cdata->src_caps = gst_caps_ref (src_caps);
509 cdata->cuda_device_id = device_id;
511 g_type_query (GST_TYPE_NV_VP8_DEC, &type_query);
512 memset (&type_info, 0, sizeof (type_info));
513 type_info.class_size = type_query.class_size;
514 type_info.instance_size = type_query.instance_size;
515 type_info.class_init = (GClassInitFunc) gst_nv_vp8_dec_subclass_init;
516 type_info.class_data = cdata;
519 type_name = g_strdup ("GstNvVP8StatelessPrimaryDec");
520 feature_name = g_strdup ("nvvp8dec");
522 type_name = g_strdup ("GstNvVP8StatelessDec");
523 feature_name = g_strdup ("nvvp8sldec");
526 if (g_type_from_name (type_name) != 0) {
528 g_free (feature_name);
531 g_strdup_printf ("GstNvVP8StatelessPrimaryDevice%dDec", device_id);
532 feature_name = g_strdup_printf ("nvvp8device%ddec", device_id);
534 type_name = g_strdup_printf ("GstNvVP8StatelessDevice%dDec", device_id);
535 feature_name = g_strdup_printf ("nvvp8sldevice%ddec", device_id);
541 cdata->is_default = is_default;
542 subtype = g_type_register_static (GST_TYPE_NV_VP8_DEC,
543 type_name, &type_info, 0);
545 /* make lower rank than default device */
546 if (rank > 0 && !is_default)
549 if (!gst_element_register (plugin, feature_name, rank, subtype))
550 GST_WARNING ("Failed to register plugin '%s'", type_name);
553 g_free (feature_name);