2 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
24 #include "gstnvvp8dec.h"
25 #include "gstcudautils.h"
26 #include "gstnvdecoder.h"
30 GST_DEBUG_CATEGORY_STATIC (gst_nv_vp8_dec_debug);
31 #define GST_CAT_DEFAULT gst_nv_vp8_dec_debug
33 /* reference list 4 + 2 margin */
34 #define NUM_OUTPUT_VIEW 6
40 GstVideoCodecState *output_state;
42 GstCudaContext *context;
43 GstNvDecoder *decoder;
44 CUVIDPICPARAMS params;
49 struct _GstNvVp8DecClass
51 GstVp8DecoderClass parent_class;
55 #define gst_nv_vp8_dec_parent_class parent_class
62 G_DEFINE_TYPE (GstNvVp8Dec, gst_nv_vp8_dec, GST_TYPE_VP8_DECODER);
64 static void gst_nv_vp8_dec_set_context (GstElement * element,
65 GstContext * context);
66 static gboolean gst_nv_vp8_dec_open (GstVideoDecoder * decoder);
67 static gboolean gst_nv_vp8_dec_close (GstVideoDecoder * decoder);
68 static gboolean gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder);
69 static gboolean gst_nv_vp8_dec_decide_allocation (GstVideoDecoder *
70 decoder, GstQuery * query);
71 static gboolean gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder,
75 static GstFlowReturn gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
76 const GstVp8FrameHdr * frame_hdr);
77 static GstFlowReturn gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
78 GstVideoCodecFrame * frame, GstVp8Picture * picture);
79 static GstFlowReturn gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
80 GstVp8Picture * picture, GstVp8Parser * parser);
81 static GstFlowReturn gst_nv_vp8_dec_output_picture (GstVp8Decoder *
82 decoder, GstVideoCodecFrame * frame, GstVp8Picture * picture);
83 static guint gst_nv_vp8_dec_get_preferred_output_delay (GstVp8Decoder * decoder,
87 gst_nv_vp8_dec_class_init (GstNvVp8DecClass * klass)
89 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
90 GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
91 GstVp8DecoderClass *vp8decoder_class = GST_VP8_DECODER_CLASS (klass);
93 element_class->set_context = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_set_context);
95 decoder_class->open = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_open);
96 decoder_class->close = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_close);
97 decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_negotiate);
98 decoder_class->decide_allocation =
99 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decide_allocation);
100 decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_src_query);
102 vp8decoder_class->new_sequence =
103 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_sequence);
104 vp8decoder_class->new_picture =
105 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_new_picture);
106 vp8decoder_class->decode_picture =
107 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_decode_picture);
108 vp8decoder_class->output_picture =
109 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_output_picture);
110 vp8decoder_class->get_preferred_output_delay =
111 GST_DEBUG_FUNCPTR (gst_nv_vp8_dec_get_preferred_output_delay);
113 GST_DEBUG_CATEGORY_INIT (gst_nv_vp8_dec_debug,
114 "nvvp8dec", 0, "NVIDIA VP8 Decoder");
116 gst_type_mark_as_plugin_api (GST_TYPE_NV_VP8_DEC, 0);
120 gst_nv_vp8_dec_init (GstNvVp8Dec * self)
125 gst_nv_vp8_dec_set_context (GstElement * element, GstContext * context)
127 GstNvVp8Dec *self = GST_NV_VP8_DEC (element);
128 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
130 GST_DEBUG_OBJECT (self, "set context %s",
131 gst_context_get_context_type (context));
133 if (gst_cuda_handle_set_context (element, context, klass->cuda_device_id,
139 gst_nv_decoder_handle_set_context (self->decoder, element, context);
142 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
146 gst_nv_vp8_dec_open (GstVideoDecoder * decoder)
148 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
149 GstNvVp8DecClass *klass = GST_NV_VP8_DEC_GET_CLASS (self);
151 if (!gst_cuda_ensure_element_context (GST_ELEMENT (self),
152 klass->cuda_device_id, &self->context)) {
153 GST_ERROR_OBJECT (self, "Required element data is unavailable");
157 self->decoder = gst_nv_decoder_new (self->context);
158 if (!self->decoder) {
159 GST_ERROR_OBJECT (self, "Failed to create decoder object");
160 gst_clear_object (&self->context);
169 gst_nv_vp8_dec_close (GstVideoDecoder * decoder)
171 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
173 g_clear_pointer (&self->output_state, gst_video_codec_state_unref);
174 gst_clear_object (&self->decoder);
175 gst_clear_object (&self->context);
181 gst_nv_vp8_dec_negotiate (GstVideoDecoder * decoder)
183 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
184 GstVp8Decoder *vp8dec = GST_VP8_DECODER (decoder);
186 GST_DEBUG_OBJECT (self, "negotiate");
188 gst_nv_decoder_negotiate (self->decoder, decoder, vp8dec->input_state,
189 &self->output_state);
191 /* TODO: add support D3D11 memory */
193 return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
197 gst_nv_vp8_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query)
199 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
201 if (!gst_nv_decoder_decide_allocation (self->decoder, decoder, query)) {
202 GST_WARNING_OBJECT (self, "Failed to handle decide allocation");
206 return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation
211 gst_nv_vp8_dec_src_query (GstVideoDecoder * decoder, GstQuery * query)
213 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
215 switch (GST_QUERY_TYPE (query)) {
216 case GST_QUERY_CONTEXT:
217 if (gst_cuda_handle_context_query (GST_ELEMENT (decoder), query,
220 } else if (self->decoder &&
221 gst_nv_decoder_handle_context_query (self->decoder, decoder, query)) {
229 return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query);
233 gst_nv_vp8_dec_new_sequence (GstVp8Decoder * decoder,
234 const GstVp8FrameHdr * frame_hdr)
236 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
237 gboolean modified = FALSE;
239 GST_LOG_OBJECT (self, "new sequence");
241 if (self->width != frame_hdr->width || self->height != frame_hdr->height) {
243 GST_INFO_OBJECT (self, "resolution changed %dx%d -> %dx%d",
244 self->width, self->height, frame_hdr->width, frame_hdr->height);
247 self->width = frame_hdr->width;
248 self->height = frame_hdr->height;
253 if (modified || !gst_nv_decoder_is_configured (self->decoder)) {
256 gst_video_info_set_format (&info,
257 GST_VIDEO_FORMAT_NV12, self->width, self->height);
259 if (!gst_nv_decoder_configure (self->decoder,
260 cudaVideoCodec_VP8, &info, self->width, self->height,
261 /* +4 for render delay */
262 NUM_OUTPUT_VIEW + 4)) {
263 GST_ERROR_OBJECT (self, "Failed to configure decoder");
264 return GST_FLOW_NOT_NEGOTIATED;
267 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
268 GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
269 return GST_FLOW_NOT_NEGOTIATED;
272 memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
274 self->params.PicWidthInMbs = GST_ROUND_UP_16 (self->width) >> 4;
275 self->params.FrameHeightInMbs = GST_ROUND_UP_16 (self->height) >> 4;
277 self->params.CodecSpecific.vp8.width = self->width;
278 self->params.CodecSpecific.vp8.height = self->height;
285 gst_nv_vp8_dec_new_picture (GstVp8Decoder * decoder,
286 GstVideoCodecFrame * frame, GstVp8Picture * picture)
288 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
289 GstNvDecoderFrame *nv_frame;
291 nv_frame = gst_nv_decoder_new_frame (self->decoder);
293 GST_ERROR_OBJECT (self, "No available decoder frame");
294 return GST_FLOW_ERROR;
297 GST_LOG_OBJECT (self,
298 "New decoder frame %p (index %d)", nv_frame, nv_frame->index);
300 gst_vp8_picture_set_user_data (picture,
301 nv_frame, (GDestroyNotify) gst_nv_decoder_frame_unref);
306 static GstNvDecoderFrame *
307 gst_nv_vp8_dec_get_decoder_frame_from_picture (GstNvVp8Dec * self,
308 GstVp8Picture * picture)
310 GstNvDecoderFrame *frame;
312 frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
315 GST_DEBUG_OBJECT (self, "current picture does not have decoder frame");
321 gst_nv_vp8_dec_decode_picture (GstVp8Decoder * decoder,
322 GstVp8Picture * picture, GstVp8Parser * parser)
324 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
325 GstVp8FrameHdr *frame_hdr = &picture->frame_hdr;
326 GstNvDecoderFrame *frame;
327 GstNvDecoderFrame *other_frame;
330 GST_LOG_OBJECT (self, "Decode picture, size %" G_GSIZE_FORMAT, picture->size);
332 frame = gst_nv_vp8_dec_get_decoder_frame_from_picture (self, picture);
334 GST_ERROR_OBJECT (self, "Decoder frame is unavailable");
335 return GST_FLOW_ERROR;
338 self->params.nBitstreamDataLen = picture->size;
339 self->params.pBitstreamData = picture->data;
340 self->params.nNumSlices = 1;
341 self->params.pSliceDataOffsets = &offset;
343 self->params.CurrPicIdx = frame->index;
345 self->params.CodecSpecific.vp8.first_partition_size =
346 frame_hdr->first_part_size;
348 if (decoder->alt_ref_picture) {
350 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
351 decoder->alt_ref_picture);
353 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for AltRef");
354 return GST_FLOW_ERROR;
357 self->params.CodecSpecific.vp8.AltRefIdx = other_frame->index;
359 self->params.CodecSpecific.vp8.AltRefIdx = 0xff;
362 if (decoder->golden_ref_picture) {
364 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
365 decoder->golden_ref_picture);
367 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for GoldenRef");
368 return GST_FLOW_ERROR;
371 self->params.CodecSpecific.vp8.GoldenRefIdx = other_frame->index;
373 self->params.CodecSpecific.vp8.GoldenRefIdx = 0xff;
376 if (decoder->last_picture) {
378 gst_nv_vp8_dec_get_decoder_frame_from_picture (self,
379 decoder->last_picture);
381 GST_ERROR_OBJECT (self, "Couldn't get decoder frame for LastRef");
382 return GST_FLOW_ERROR;
385 self->params.CodecSpecific.vp8.LastRefIdx = other_frame->index;
387 self->params.CodecSpecific.vp8.LastRefIdx = 0xff;
390 self->params.CodecSpecific.vp8.vp8_frame_tag.frame_type =
391 frame_hdr->key_frame ? 0 : 1;
392 self->params.CodecSpecific.vp8.vp8_frame_tag.version = frame_hdr->version;
393 self->params.CodecSpecific.vp8.vp8_frame_tag.show_frame =
394 frame_hdr->show_frame;
395 self->params.CodecSpecific.vp8.vp8_frame_tag.update_mb_segmentation_data =
396 parser->segmentation.segmentation_enabled ?
397 parser->segmentation.update_segment_feature_data : 0;
399 if (!gst_nv_decoder_decode_picture (self->decoder, &self->params))
400 return GST_FLOW_ERROR;
406 gst_nv_vp8_dec_output_picture (GstVp8Decoder * decoder,
407 GstVideoCodecFrame * frame, GstVp8Picture * picture)
409 GstNvVp8Dec *self = GST_NV_VP8_DEC (decoder);
410 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
411 GstNvDecoderFrame *decoder_frame;
413 GST_LOG_OBJECT (self, "Outputting picture %p", picture);
415 decoder_frame = (GstNvDecoderFrame *) gst_vp8_picture_get_user_data (picture);
416 if (!decoder_frame) {
417 GST_ERROR_OBJECT (self, "No decoder frame in picture %p", picture);
421 if (!gst_nv_decoder_finish_frame (self->decoder, vdec, decoder_frame,
422 &frame->output_buffer)) {
423 GST_ERROR_OBJECT (self, "Failed to handle output picture");
427 gst_vp8_picture_unref (picture);
429 return gst_video_decoder_finish_frame (vdec, frame);
432 gst_video_decoder_drop_frame (vdec, frame);
433 gst_vp8_picture_unref (picture);
435 return GST_FLOW_ERROR;
439 gst_nv_vp8_dec_get_preferred_output_delay (GstVp8Decoder * decoder,
442 /* Prefer to zero latency for live pipeline */
446 /* NVCODEC SDK uses 4 frame delay for better throughput performance */
454 guint cuda_device_id;
456 } GstNvVp8DecClassData;
459 gst_nv_vp8_dec_subclass_init (gpointer klass, GstNvVp8DecClassData * cdata)
461 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
462 GstNvVp8DecClass *nvdec_class = (GstNvVp8DecClass *) (klass);
465 if (cdata->is_default) {
466 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder");
468 long_name = g_strdup_printf ("NVDEC VP8 Stateless Decoder with device %d",
469 cdata->cuda_device_id);
472 gst_element_class_set_metadata (element_class, long_name,
473 "Codec/Decoder/Video/Hardware",
474 "NVIDIA VP8 video decoder", "Seungha Yang <seungha@centricular.com>");
477 gst_element_class_add_pad_template (element_class,
478 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
480 gst_element_class_add_pad_template (element_class,
481 gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
484 nvdec_class->cuda_device_id = cdata->cuda_device_id;
486 gst_caps_unref (cdata->sink_caps);
487 gst_caps_unref (cdata->src_caps);
492 gst_nv_vp8_dec_register (GstPlugin * plugin, guint device_id, guint rank,
493 GstCaps * sink_caps, GstCaps * src_caps, gboolean is_primary)
495 GTypeQuery type_query;
496 GTypeInfo type_info = { 0, };
500 GstNvVp8DecClassData *cdata;
501 gboolean is_default = TRUE;
504 * element-nvvp8sldec:
509 cdata = g_new0 (GstNvVp8DecClassData, 1);
510 cdata->sink_caps = gst_caps_ref (sink_caps);
511 cdata->src_caps = gst_caps_ref (src_caps);
512 cdata->cuda_device_id = device_id;
514 g_type_query (GST_TYPE_NV_VP8_DEC, &type_query);
515 memset (&type_info, 0, sizeof (type_info));
516 type_info.class_size = type_query.class_size;
517 type_info.instance_size = type_query.instance_size;
518 type_info.class_init = (GClassInitFunc) gst_nv_vp8_dec_subclass_init;
519 type_info.class_data = cdata;
522 type_name = g_strdup ("GstNvVP8StatelessPrimaryDec");
523 feature_name = g_strdup ("nvvp8dec");
525 type_name = g_strdup ("GstNvVP8StatelessDec");
526 feature_name = g_strdup ("nvvp8sldec");
529 if (g_type_from_name (type_name) != 0) {
531 g_free (feature_name);
534 g_strdup_printf ("GstNvVP8StatelessPrimaryDevice%dDec", device_id);
535 feature_name = g_strdup_printf ("nvvp8device%ddec", device_id);
537 type_name = g_strdup_printf ("GstNvVP8StatelessDevice%dDec", device_id);
538 feature_name = g_strdup_printf ("nvvp8sldevice%ddec", device_id);
544 cdata->is_default = is_default;
545 subtype = g_type_register_static (GST_TYPE_NV_VP8_DEC,
546 type_name, &type_info, 0);
548 /* make lower rank than default device */
549 if (rank > 0 && !is_default)
552 if (!gst_element_register (plugin, feature_name, rank, subtype))
553 GST_WARNING ("Failed to register plugin '%s'", type_name);
556 g_free (feature_name);