2 * Copyright (C) 2020 Igalia, S.L.
3 * Author: Víctor Jáquez <vjaquez@igalia.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the0
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-vavp9dec
24 * @short_description: A VA-API based VP9 video decoder
26 * vavp9dec decodes VP9 bitstreams to VA surfaces using the
27 * installed and chosen [VA-API](https://01.org/linuxmedia/vaapi)
30 * The decoding surfaces can be mapped onto main memory as video
33 * ## Example launch line
35 * gst-launch-1.0 filesrc location=sample.webm ! parsebin ! vavp9dec ! autovideosink
46 #include "gstvavp9dec.h"
48 #include "gstvabasedec.h"
50 GST_DEBUG_CATEGORY_STATIC (gst_va_vp9dec_debug);
51 #ifndef GST_DISABLE_GST_DEBUG
52 #define GST_CAT_DEFAULT gst_va_vp9dec_debug
54 #define GST_CAT_DEFAULT NULL
57 #define GST_VA_VP9_DEC(obj) ((GstVaVp9Dec *) obj)
58 #define GST_VA_VP9_DEC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), G_TYPE_FROM_INSTANCE (obj), GstVaVp9DecClass))
59 #define GST_VA_VP9_DEC_CLASS(klass) ((GstVaVp9DecClass *) klass)
61 typedef struct _GstVaVp9Dec GstVaVp9Dec;
62 typedef struct _GstVaVp9DecClass GstVaVp9DecClass;
64 struct _GstVaVp9DecClass
66 GstVaBaseDecClass parent_class;
73 gboolean need_negotiation;
76 #define parent_class gst_va_base_dec_parent_class
77 extern gpointer gst_va_base_dec_parent_class;
80 static const gchar *src_caps_str = GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("memory:VAMemory",
81 "{ NV12 }") " ;" GST_VIDEO_CAPS_MAKE ("{ NV12 }");
84 static const gchar *sink_caps_str = "video/x-vp9";
87 _get_rtformat (GstVaVp9Dec * self, GstVP9Profile profile,
88 GstVp9BitDepth bit_depth, gint subsampling_x, gint subsampling_y)
91 case GST_VP9_PROFILE_0:
92 return VA_RT_FORMAT_YUV420;
93 case GST_VP9_PROFILE_1:
94 if (subsampling_x == 1 && subsampling_y == 0)
95 return VA_RT_FORMAT_YUV422;
96 else if (subsampling_x == 0 && subsampling_y == 0)
97 return VA_RT_FORMAT_YUV444;
99 case GST_VP9_PROFILE_2:
100 if (bit_depth == GST_VP9_BIT_DEPTH_10)
101 return VA_RT_FORMAT_YUV420_10;
102 else if (bit_depth == GST_VP9_BIT_DEPTH_12)
103 return VA_RT_FORMAT_YUV420_12;
105 case GST_VP9_PROFILE_3:
106 if (subsampling_x == 1 && subsampling_y == 0) {
107 if (bit_depth == GST_VP9_BIT_DEPTH_10)
108 return VA_RT_FORMAT_YUV422_10;
109 else if (bit_depth == GST_VP9_BIT_DEPTH_12)
110 return VA_RT_FORMAT_YUV422_12;
111 } else if (subsampling_x == 0 && subsampling_y == 0) {
112 if (bit_depth == GST_VP9_BIT_DEPTH_10)
113 return VA_RT_FORMAT_YUV444_10;
114 else if (bit_depth == GST_VP9_BIT_DEPTH_12)
115 return VA_RT_FORMAT_YUV444_12;
122 GST_ERROR_OBJECT (self, "Unsupported chroma format");
127 _get_profile (GstVaVp9Dec * self, GstVP9Profile profile)
130 case GST_VP9_PROFILE_0:
131 return VAProfileVP9Profile0;
132 case GST_VP9_PROFILE_1:
133 return VAProfileVP9Profile1;
134 case GST_VP9_PROFILE_2:
135 return VAProfileVP9Profile2;
136 case GST_VP9_PROFILE_3:
137 return VAProfileVP9Profile3;
142 GST_ERROR_OBJECT (self, "Unsupported profile");
143 return VAProfileNone;
147 gst_va_vp9_new_sequence (GstVp9Decoder * decoder, const GstVp9Parser * parser,
148 const GstVp9FrameHdr * frame_hdr)
150 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
151 GstVaVp9Dec *self = GST_VA_VP9_DEC (decoder);
153 gboolean negotiation_needed = FALSE;
156 profile = _get_profile (self, frame_hdr->profile);
157 if (profile == VAProfileNone)
160 if (!gst_va_decoder_has_profile (base->decoder, profile)) {
161 GST_ERROR_OBJECT (self, "Profile %s is not supported",
162 gst_va_profile_name (profile));
166 rt_format = _get_rtformat (self, frame_hdr->profile, parser->bit_depth,
167 parser->subsampling_x, parser->subsampling_y);
171 if (gst_va_decoder_format_changed (base->decoder, profile,
172 rt_format, frame_hdr->width, frame_hdr->height)) {
173 base->profile = profile;
174 base->width = frame_hdr->width;
175 base->height = frame_hdr->height;
176 base->rt_format = rt_format;
177 negotiation_needed = TRUE;
180 base->min_buffers = GST_VP9_REF_FRAMES;
182 if (negotiation_needed) {
183 self->need_negotiation = TRUE;
184 if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) {
185 GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
194 gst_va_vp9_dec_new_picture (GstVp9Decoder * decoder,
195 GstVideoCodecFrame * frame, GstVp9Picture * picture)
198 GstVaVp9Dec *self = GST_VA_VP9_DEC (decoder);
199 GstVaDecodePicture *pic;
200 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
201 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
203 ret = gst_video_decoder_allocate_output_frame (vdec, frame);
204 if (ret != GST_FLOW_OK)
207 pic = gst_va_decode_picture_new (base->decoder, frame->output_buffer);
209 gst_vp9_picture_set_user_data (picture, pic,
210 (GDestroyNotify) gst_va_decode_picture_free);
212 GST_LOG_OBJECT (self, "New va decode picture %p - %#x", pic,
213 gst_va_decode_picture_get_surface (pic));
219 GST_WARNING_OBJECT (self, "Failed to allocated output buffer, return %s",
220 gst_flow_get_name (ret));
225 static inline gboolean
226 _fill_param (GstVp9Decoder * decoder, GstVp9Picture * picture, GstVp9Dpb * dpb)
228 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
229 GstVaDecodePicture *va_pic;
230 const GstVp9FrameHdr *frame_hdr = &picture->frame_hdr;
231 const GstVp9LoopFilter *loopfilter = &frame_hdr->loopfilter;
232 const GstVp9SegmentationInfo *seg = &frame_hdr->segmentation;
233 VADecPictureParameterBufferVP9 pic_param;
237 pic_param = (VADecPictureParameterBufferVP9) {
238 .frame_width = base->width,
239 .frame_height = base->height,
242 .subsampling_x = picture->subsampling_x,
243 .subsampling_y = picture->subsampling_x,
244 .frame_type = frame_hdr->frame_type,
245 .show_frame = frame_hdr->show_frame,
246 .error_resilient_mode = frame_hdr->error_resilient_mode,
247 .intra_only = frame_hdr->intra_only,
248 .allow_high_precision_mv = (frame_hdr->frame_type == GST_VP9_KEY_FRAME) ?
249 0 : frame_hdr->allow_high_precision_mv,
250 .mcomp_filter_type = frame_hdr->mcomp_filter_type,
251 .frame_parallel_decoding_mode = frame_hdr->frame_parallel_decoding_mode,
252 .reset_frame_context = frame_hdr->reset_frame_context,
253 .refresh_frame_context = frame_hdr->refresh_frame_context,
254 .frame_context_idx = frame_hdr->frame_context_idx,
256 .segmentation_enabled = seg->enabled,
257 .segmentation_temporal_update = seg->temporal_update,
258 .segmentation_update_map = seg->update_map,
261 frame_hdr->ref_frame_indices[GST_VP9_REF_FRAME_LAST - 1],
262 .last_ref_frame_sign_bias =
263 frame_hdr->ref_frame_sign_bias[GST_VP9_REF_FRAME_LAST - 1],
265 frame_hdr->ref_frame_indices[GST_VP9_REF_FRAME_GOLDEN - 1],
266 .golden_ref_frame_sign_bias =
267 frame_hdr->ref_frame_sign_bias[GST_VP9_REF_FRAME_GOLDEN - 1],
269 frame_hdr->ref_frame_indices[GST_VP9_REF_FRAME_ALTREF - 1],
270 .alt_ref_frame_sign_bias =
271 frame_hdr->ref_frame_sign_bias[GST_VP9_REF_FRAME_ALTREF - 1],
273 .lossless_flag = frame_hdr->lossless_flag,
276 .filter_level = loopfilter->filter_level,
277 .sharpness_level = loopfilter->sharpness_level,
278 .log2_tile_rows = frame_hdr->log2_tile_rows,
279 .log2_tile_columns = frame_hdr->log2_tile_columns,
281 .frame_header_length_in_bytes = frame_hdr->frame_header_length_in_bytes,
282 .first_partition_size = frame_hdr->first_partition_size,
284 .profile = frame_hdr->profile,
285 .bit_depth = picture->bit_depth
289 memcpy (pic_param.mb_segment_tree_probs, seg->tree_probs,
290 sizeof (seg->tree_probs));
292 if (seg->temporal_update) {
293 memcpy (pic_param.segment_pred_probs, seg->pred_probs,
294 sizeof (seg->pred_probs));
296 memset (pic_param.segment_pred_probs, 255,
297 sizeof (pic_param.segment_pred_probs));
300 for (i = 0; i < GST_VP9_REF_FRAMES; i++) {
301 if (dpb->pic_list[i]) {
302 GstVaDecodePicture *va_pic =
303 gst_vp9_picture_get_user_data (dpb->pic_list[i]);
305 pic_param.reference_frames[i] =
306 gst_va_decode_picture_get_surface (va_pic);
308 pic_param.reference_frames[i] = VA_INVALID_ID;
312 va_pic = gst_vp9_picture_get_user_data (picture);
314 return gst_va_decoder_add_param_buffer (base->decoder, va_pic,
315 VAPictureParameterBufferType, &pic_param, sizeof (pic_param));
318 static inline gboolean
319 _fill_slice (GstVp9Decoder * decoder, GstVp9Picture * picture)
321 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
322 GstVaDecodePicture *va_pic;
323 const GstVp9Segmentation *seg = picture->segmentation;
324 VASliceParameterBufferVP9 slice_param;
328 slice_param = (VASliceParameterBufferVP9) {
329 .slice_data_size = picture->size,
330 .slice_data_offset = 0,
331 .slice_data_flag = VA_SLICE_DATA_FLAG_ALL,
335 for (i = 0; i < GST_VP9_MAX_SEGMENTS; i++) {
337 slice_param.seg_param[i] = (VASegmentParameterVP9) {
338 .segment_flags.fields = {
339 .segment_reference_enabled = seg[i].reference_frame_enabled,
340 .segment_reference = seg[i].reference_frame,
341 .segment_reference_skipped = seg[i].reference_skip,
344 .luma_dc_quant_scale = seg[i].luma_dc_quant_scale,
345 .luma_ac_quant_scale = seg[i].luma_ac_quant_scale,
346 .chroma_dc_quant_scale = seg[i].chroma_dc_quant_scale,
347 .chroma_ac_quant_scale = seg[i].chroma_ac_quant_scale,
351 memcpy (slice_param.seg_param[i].filter_level, seg[i].filter_level,
352 sizeof (slice_param.seg_param[i].filter_level));
355 va_pic = gst_vp9_picture_get_user_data (picture);
357 return gst_va_decoder_add_slice_buffer (base->decoder, va_pic, &slice_param,
358 sizeof (slice_param), (gpointer) picture->data, picture->size);
363 gst_va_vp9_decode_picture (GstVp9Decoder * decoder, GstVp9Picture * picture,
366 return _fill_param (decoder, picture, dpb) && _fill_slice (decoder, picture);
370 gst_va_vp9_dec_end_picture (GstVp9Decoder * decoder, GstVp9Picture * picture)
372 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
373 GstVaDecodePicture *va_pic;
375 GST_LOG_OBJECT (base, "end picture %p", picture);
377 va_pic = gst_vp9_picture_get_user_data (picture);
379 return gst_va_decoder_decode (base->decoder, va_pic);
383 gst_va_vp9_dec_output_picture (GstVp9Decoder * decoder,
384 GstVideoCodecFrame * frame, GstVp9Picture * picture)
386 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
387 GstVaVp9Dec *self = GST_VA_VP9_DEC (decoder);
389 GST_LOG_OBJECT (self, "Outputting picture %p", picture);
391 if (base->copy_frames)
392 gst_va_base_dec_copy_output_buffer (base, frame);
394 gst_vp9_picture_unref (picture);
396 return gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame);
399 static GstVp9Picture *
400 gst_va_vp9_dec_duplicate_picture (GstVp9Decoder * decoder,
401 GstVp9Picture * picture)
403 GstVaDecodePicture *va_pic, *va_dup;
404 GstVp9Picture *new_picture;
406 va_pic = gst_vp9_picture_get_user_data (picture);
407 va_dup = gst_va_decode_picture_dup (va_pic);
409 new_picture = gst_vp9_picture_new ();
410 new_picture->frame_hdr = picture->frame_hdr;
412 gst_vp9_picture_set_user_data (picture, va_dup,
413 (GDestroyNotify) gst_va_decode_picture_free);
419 gst_va_vp9_dec_negotiate (GstVideoDecoder * decoder)
421 GstCapsFeatures *capsfeatures = NULL;
422 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
423 GstVaVp9Dec *self = GST_VA_VP9_DEC (decoder);
424 GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
425 GstVp9Decoder *vp9dec = GST_VP9_DECODER (decoder);
427 /* Ignore downstream renegotiation request. */
428 if (!self->need_negotiation)
431 self->need_negotiation = FALSE;
433 if (gst_va_decoder_is_open (base->decoder)
434 && !gst_va_decoder_close (base->decoder))
437 if (!gst_va_decoder_open (base->decoder, base->profile, base->rt_format))
440 if (!gst_va_decoder_set_format (base->decoder, base->width, base->height,
444 if (base->output_state)
445 gst_video_codec_state_unref (base->output_state);
447 gst_va_base_dec_get_preferred_format_and_caps_features (base, &format,
451 gst_video_decoder_set_output_state (decoder, format,
452 base->width, base->height, vp9dec->input_state);
454 base->output_state->caps = gst_video_info_to_caps (&base->output_state->info);
456 gst_caps_set_features_simple (base->output_state->caps, capsfeatures);
458 GST_INFO_OBJECT (self, "Negotiated caps %" GST_PTR_FORMAT,
459 base->output_state->caps);
461 return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder);
465 gst_va_vp9_dec_dispose (GObject * object)
467 gst_va_base_dec_close (GST_VIDEO_DECODER (object));
468 G_OBJECT_CLASS (parent_class)->dispose (object);
472 gst_va_vp9_dec_class_init (gpointer g_class, gpointer class_data)
474 GstCaps *src_doc_caps, *sink_doc_caps;
475 GObjectClass *gobject_class = G_OBJECT_CLASS (g_class);
476 GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
477 GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (g_class);
478 GstVp9DecoderClass *vp9_class = GST_VP9_DECODER_CLASS (g_class);
479 struct CData *cdata = class_data;
482 if (cdata->description) {
483 long_name = g_strdup_printf ("VA-API VP9 Decoder in %s",
486 long_name = g_strdup ("VA-API VP9 Decoder");
489 gst_element_class_set_metadata (element_class, long_name,
490 "Codec/Decoder/Video/Hardware", "VA-API based VP9 video decoder",
491 "Víctor Jáquez <vjaquez@igalia.com>");
493 sink_doc_caps = gst_caps_from_string (sink_caps_str);
494 src_doc_caps = gst_caps_from_string (src_caps_str);
496 gst_va_base_dec_class_init (GST_VA_BASE_DEC_CLASS (g_class), VP9,
497 cdata->render_device_path, cdata->sink_caps, cdata->src_caps,
498 src_doc_caps, sink_doc_caps);
500 gobject_class->dispose = gst_va_vp9_dec_dispose;
502 decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_va_vp9_dec_negotiate);
504 vp9_class->new_sequence = GST_DEBUG_FUNCPTR (gst_va_vp9_new_sequence);
505 vp9_class->new_picture = GST_DEBUG_FUNCPTR (gst_va_vp9_dec_new_picture);
506 vp9_class->decode_picture = GST_DEBUG_FUNCPTR (gst_va_vp9_decode_picture);
507 vp9_class->end_picture = GST_DEBUG_FUNCPTR (gst_va_vp9_dec_end_picture);
508 vp9_class->output_picture = GST_DEBUG_FUNCPTR (gst_va_vp9_dec_output_picture);
509 vp9_class->duplicate_picture =
510 GST_DEBUG_FUNCPTR (gst_va_vp9_dec_duplicate_picture);
513 g_free (cdata->description);
514 g_free (cdata->render_device_path);
515 gst_caps_unref (cdata->src_caps);
516 gst_caps_unref (cdata->sink_caps);
521 gst_va_vp9_dec_init (GTypeInstance * instance, gpointer g_class)
523 gst_va_base_dec_init (GST_VA_BASE_DEC (instance), GST_CAT_DEFAULT);
526 /* This element doesn't parse supreframes. Let's delegate it to the
529 _complete_sink_caps (GstCaps * sinkcaps)
531 gst_caps_set_simple (sinkcaps, "alignment", G_TYPE_STRING, "frame", NULL);
532 return gst_caps_ref (sinkcaps);
536 _register_debug_category (gpointer data)
538 GST_DEBUG_CATEGORY_INIT (gst_va_vp9dec_debug, "vavp9dec", 0,
545 gst_va_vp9_dec_register (GstPlugin * plugin, GstVaDevice * device,
546 GstCaps * sink_caps, GstCaps * src_caps, guint rank)
548 static GOnce debug_once = G_ONCE_INIT;
550 GTypeInfo type_info = {
551 .class_size = sizeof (GstVaVp9DecClass),
552 .class_init = gst_va_vp9_dec_class_init,
553 .instance_size = sizeof (GstVaVp9Dec),
554 .instance_init = gst_va_vp9_dec_init,
558 gchar *type_name, *feature_name;
560 g_return_val_if_fail (GST_IS_PLUGIN (plugin), FALSE);
561 g_return_val_if_fail (GST_IS_VA_DEVICE (device), FALSE);
562 g_return_val_if_fail (GST_IS_CAPS (sink_caps), FALSE);
563 g_return_val_if_fail (GST_IS_CAPS (src_caps), FALSE);
565 cdata = g_new (struct CData, 1);
566 cdata->description = NULL;
567 cdata->render_device_path = g_strdup (device->render_device_path);
568 cdata->sink_caps = _complete_sink_caps (sink_caps);
569 cdata->src_caps = gst_caps_ref (src_caps);
571 /* class data will be leaked if the element never gets instantiated */
572 GST_MINI_OBJECT_FLAG_SET (sink_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
573 GST_MINI_OBJECT_FLAG_SET (src_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
575 type_info.class_data = cdata;
577 type_name = g_strdup ("GstVaVp9Dec");
578 feature_name = g_strdup ("vavp9dec");
580 /* The first decoder to be registered should use a constant name,
581 * like vavp9dec, for any additional decoders, we create unique
582 * names, using inserting the render device name. */
583 if (g_type_from_name (type_name)) {
584 gchar *basename = g_path_get_basename (device->render_device_path);
586 g_free (feature_name);
587 type_name = g_strdup_printf ("GstVa%sVp9Dec", basename);
588 feature_name = g_strdup_printf ("va%svp9dec", basename);
589 cdata->description = basename;
591 /* lower rank for non-first device */
596 g_once (&debug_once, _register_debug_category, NULL);
598 type = g_type_register_static (GST_TYPE_VP9_DECODER,
599 type_name, &type_info, 0);
601 ret = gst_element_register (plugin, feature_name, rank, type);
604 g_free (feature_name);