2 * Copyright (C) 2020 Intel Corporation
3 * Author: He Junyan <junyan.he@intel.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the0
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-vavp8dec
24 * @short_description: A VA-API based VP8 video decoder
26 * vavp8dec decodes VP8 bitstreams to VA surfaces using the
27 * installed and chosen [VA-API](https://01.org/linuxmedia/vaapi)
30 * The decoding surfaces can be mapped onto main memory as video
33 * ## Example launch line
35 * gst-launch-1.0 filesrc location=sample.webm ! parsebin ! vavp8dec ! autovideosink
46 #include "gstvavp8dec.h"
48 #include "gstvabasedec.h"
50 GST_DEBUG_CATEGORY_STATIC (gst_va_vp8dec_debug);
51 #ifndef GST_DISABLE_GST_DEBUG
52 #define GST_CAT_DEFAULT gst_va_vp8dec_debug
54 #define GST_CAT_DEFAULT NULL
57 #define GST_VA_VP8_DEC(obj) ((GstVaVp8Dec *) obj)
58 #define GST_VA_VP8_DEC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), G_TYPE_FROM_INSTANCE (obj), GstVaVp8DecClass))
59 #define GST_VA_VP8_DEC_CLASS(klass) ((GstVaVp8DecClass *) klass)
61 typedef struct _GstVaVp8Dec GstVaVp8Dec;
62 typedef struct _GstVaVp8DecClass GstVaVp8DecClass;
64 struct _GstVaVp8DecClass
66 GstVaBaseDecClass parent_class;
74 static GstElementClass *parent_class = NULL;
77 static const gchar *src_caps_str =
78 GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VA,
80 GST_VIDEO_CAPS_MAKE ("{ NV12 }");
83 static const gchar *sink_caps_str = "video/x-vp8";
86 _get_profile (GstVaVp8Dec * self, const GstVp8FrameHdr * frame_hdr)
89 if (frame_hdr->version > 3) {
90 GST_ERROR_OBJECT (self, "Unsupported vp8 version: %d", frame_hdr->version);
94 return VAProfileVP8Version0_3;
98 gst_va_vp8_dec_new_sequence (GstVp8Decoder * decoder,
99 const GstVp8FrameHdr * frame_hdr, gint max_dpb_size)
101 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
102 GstVaVp8Dec *self = GST_VA_VP8_DEC (decoder);
103 GstVideoInfo *info = &base->output_info;
106 gboolean negotiation_needed = FALSE;
108 GST_LOG_OBJECT (self, "new sequence");
110 profile = _get_profile (self, frame_hdr);
111 if (profile == VAProfileNone)
112 return GST_FLOW_NOT_NEGOTIATED;
114 if (!gst_va_decoder_has_profile (base->decoder, profile)) {
115 GST_ERROR_OBJECT (self, "Profile %s is not supported",
116 gst_va_profile_name (profile));
117 return GST_FLOW_NOT_NEGOTIATED;
120 /* VP8 always use 8 bits 4:2:0 */
121 rt_format = VA_RT_FORMAT_YUV420;
123 if (!gst_va_decoder_config_is_equal (base->decoder, profile,
124 rt_format, frame_hdr->width, frame_hdr->height)) {
125 base->profile = profile;
126 GST_VIDEO_INFO_WIDTH (info) = base->width = frame_hdr->width;
127 GST_VIDEO_INFO_HEIGHT (info) = base->height = frame_hdr->height;
128 base->rt_format = rt_format;
129 negotiation_needed = TRUE;
132 base->min_buffers = 3 + 4; /* max num pic references + scratch surfaces */
133 base->need_negotiation = negotiation_needed;
134 g_clear_pointer (&base->input_state, gst_video_codec_state_unref);
135 base->input_state = gst_video_codec_state_ref (decoder->input_state);
141 gst_va_vp8_dec_new_picture (GstVp8Decoder * decoder,
142 GstVideoCodecFrame * frame, GstVp8Picture * picture)
144 GstVaVp8Dec *self = GST_VA_VP8_DEC (decoder);
145 GstVaDecodePicture *pic;
146 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
149 ret = gst_va_base_dec_prepare_output_frame (base, frame);
150 if (ret != GST_FLOW_OK)
153 pic = gst_va_decode_picture_new (base->decoder, frame->output_buffer);
155 gst_vp8_picture_set_user_data (picture, pic,
156 (GDestroyNotify) gst_va_decode_picture_free);
158 GST_LOG_OBJECT (self, "New va decode picture %p - %#x", pic,
159 gst_va_decode_picture_get_surface (pic));
165 GST_WARNING_OBJECT (self,
166 "Failed to allocated output buffer, return %s",
167 gst_flow_get_name (ret));
173 _fill_quant_matrix (GstVp8Decoder * decoder, GstVp8Picture * picture,
174 GstVp8Parser * parser)
176 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
177 GstVp8FrameHdr const *frame_hdr = &picture->frame_hdr;
178 GstVp8Segmentation *const seg = &parser->segmentation;
179 VAIQMatrixBufferVP8 iq_matrix = { 0, };
180 const gint8 QI_MAX = 127;
184 /* Fill in VAIQMatrixBufferVP8 */
185 for (i = 0; i < 4; i++) {
186 if (seg->segmentation_enabled) {
187 qi_base = seg->quantizer_update_value[i];
188 if (!seg->segment_feature_mode) /* 0 means delta update */
189 qi_base += frame_hdr->quant_indices.y_ac_qi;
191 qi_base = frame_hdr->quant_indices.y_ac_qi;
194 iq_matrix.quantization_index[i][0] = CLAMP (qi, 0, QI_MAX);
195 qi = qi_base + frame_hdr->quant_indices.y_dc_delta;
196 iq_matrix.quantization_index[i][1] = CLAMP (qi, 0, QI_MAX);
197 qi = qi_base + frame_hdr->quant_indices.y2_dc_delta;
198 iq_matrix.quantization_index[i][2] = CLAMP (qi, 0, QI_MAX);
199 qi = qi_base + frame_hdr->quant_indices.y2_ac_delta;
200 iq_matrix.quantization_index[i][3] = CLAMP (qi, 0, QI_MAX);
201 qi = qi_base + frame_hdr->quant_indices.uv_dc_delta;
202 iq_matrix.quantization_index[i][4] = CLAMP (qi, 0, QI_MAX);
203 qi = qi_base + frame_hdr->quant_indices.uv_ac_delta;
204 iq_matrix.quantization_index[i][5] = CLAMP (qi, 0, QI_MAX);
207 return gst_va_decoder_add_param_buffer (base->decoder,
208 gst_vp8_picture_get_user_data (picture), VAIQMatrixBufferType, &iq_matrix,
213 _fill_probability_table (GstVp8Decoder * decoder, GstVp8Picture * picture)
215 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
216 GstVp8FrameHdr const *frame_hdr = &picture->frame_hdr;
217 VAProbabilityDataBufferVP8 prob_table = { 0, };
219 /* Fill in VAProbabilityDataBufferVP8 */
220 memcpy (prob_table.dct_coeff_probs, frame_hdr->token_probs.prob,
221 sizeof (frame_hdr->token_probs.prob));
223 return gst_va_decoder_add_param_buffer (base->decoder,
224 gst_vp8_picture_get_user_data (picture), VAProbabilityBufferType,
225 &prob_table, sizeof (prob_table));
229 _fill_picture (GstVp8Decoder * decoder, GstVp8Picture * picture,
230 GstVp8Parser * parser)
232 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
233 GstVaDecodePicture *va_pic;
234 VAPictureParameterBufferVP8 pic_param;
235 GstVp8FrameHdr const *frame_hdr = &picture->frame_hdr;
236 GstVp8Segmentation *const seg = &parser->segmentation;
239 if (!_fill_quant_matrix (decoder, picture, parser))
242 if (!_fill_probability_table (decoder, picture))
246 pic_param = (VAPictureParameterBufferVP8) {
247 .frame_width = base->width,
248 .frame_height = base->height,
249 .last_ref_frame = VA_INVALID_SURFACE,
250 .golden_ref_frame = VA_INVALID_SURFACE,
251 .alt_ref_frame = VA_INVALID_SURFACE,
252 .out_of_loop_frame = VA_INVALID_SURFACE, // not used currently
253 .pic_fields.bits.key_frame = !frame_hdr->key_frame,
254 .pic_fields.bits.version = frame_hdr->version,
255 .pic_fields.bits.segmentation_enabled = seg->segmentation_enabled,
256 .pic_fields.bits.update_mb_segmentation_map =
257 seg->update_mb_segmentation_map,
258 .pic_fields.bits.update_segment_feature_data =
259 seg->update_segment_feature_data,
260 .pic_fields.bits.filter_type = frame_hdr->filter_type,
261 .pic_fields.bits.sharpness_level = frame_hdr->sharpness_level,
262 .pic_fields.bits.loop_filter_adj_enable =
263 parser->mb_lf_adjust.loop_filter_adj_enable,
264 .pic_fields.bits.mode_ref_lf_delta_update =
265 parser->mb_lf_adjust.mode_ref_lf_delta_update,
266 .pic_fields.bits.sign_bias_golden = frame_hdr->sign_bias_golden,
267 .pic_fields.bits.sign_bias_alternate = frame_hdr->sign_bias_alternate,
268 .pic_fields.bits.mb_no_coeff_skip = frame_hdr->mb_no_skip_coeff,
269 /* In decoding, the only loop filter settings that matter are those
270 in the frame header (9.1) */
271 .pic_fields.bits.loop_filter_disable = frame_hdr->loop_filter_level == 0,
272 .prob_skip_false = frame_hdr->prob_skip_false,
273 .prob_intra = frame_hdr->prob_intra,
274 .prob_last = frame_hdr->prob_last,
275 .prob_gf = frame_hdr->prob_gf,
276 .bool_coder_ctx.range = frame_hdr->rd_range,
277 .bool_coder_ctx.value = frame_hdr->rd_value,
278 .bool_coder_ctx.count = frame_hdr->rd_count,
282 if (!frame_hdr->key_frame) {
283 if (decoder->last_picture) {
284 va_pic = gst_vp8_picture_get_user_data (decoder->last_picture);
285 pic_param.last_ref_frame = gst_va_decode_picture_get_surface (va_pic);
287 if (decoder->golden_ref_picture) {
288 va_pic = gst_vp8_picture_get_user_data (decoder->golden_ref_picture);
289 pic_param.golden_ref_frame = gst_va_decode_picture_get_surface (va_pic);
291 if (decoder->alt_ref_picture) {
292 va_pic = gst_vp8_picture_get_user_data (decoder->alt_ref_picture);
293 pic_param.alt_ref_frame = gst_va_decode_picture_get_surface (va_pic);
297 for (i = 0; i < 3; i++)
298 pic_param.mb_segment_tree_probs[i] = seg->segment_prob[i];
300 for (i = 0; i < 4; i++) {
302 if (seg->segmentation_enabled) {
303 level = seg->lf_update_value[i];
304 /* 0 means delta update */
305 if (!seg->segment_feature_mode)
306 level += frame_hdr->loop_filter_level;
308 level = frame_hdr->loop_filter_level;
309 pic_param.loop_filter_level[i] = CLAMP (level, 0, 63);
311 pic_param.loop_filter_deltas_ref_frame[i] =
312 parser->mb_lf_adjust.ref_frame_delta[i];
313 pic_param.loop_filter_deltas_mode[i] =
314 parser->mb_lf_adjust.mb_mode_delta[i];
317 memcpy (pic_param.y_mode_probs, frame_hdr->mode_probs.y_prob,
318 sizeof (frame_hdr->mode_probs.y_prob));
319 memcpy (pic_param.uv_mode_probs, frame_hdr->mode_probs.uv_prob,
320 sizeof (frame_hdr->mode_probs.uv_prob));
321 memcpy (pic_param.mv_probs, frame_hdr->mv_probs.prob,
322 sizeof (frame_hdr->mv_probs));
324 va_pic = gst_vp8_picture_get_user_data (picture);
325 return gst_va_decoder_add_param_buffer (base->decoder, va_pic,
326 VAPictureParameterBufferType, &pic_param, sizeof (pic_param));
330 _add_slice (GstVp8Decoder * decoder, GstVp8Picture * picture,
331 GstVp8Parser * parser)
333 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
334 GstVp8FrameHdr const *frame_hdr = &picture->frame_hdr;
335 VASliceParameterBufferVP8 slice_param;
336 GstVaDecodePicture *va_pic;
340 slice_param = (VASliceParameterBufferVP8) {
341 .slice_data_size = picture->size,
342 .slice_data_offset = frame_hdr->data_chunk_size,
343 .macroblock_offset = frame_hdr->header_size,
344 .num_of_partitions = (1 << frame_hdr->log2_nbr_of_dct_partitions) + 1,
348 slice_param.partition_size[0] =
349 frame_hdr->first_part_size - ((slice_param.macroblock_offset + 7) >> 3);
350 for (i = 1; i < slice_param.num_of_partitions; i++)
351 slice_param.partition_size[i] = frame_hdr->partition_size[i - 1];
352 for (; i < G_N_ELEMENTS (slice_param.partition_size); i++)
353 slice_param.partition_size[i] = 0;
355 va_pic = gst_vp8_picture_get_user_data (picture);
356 return gst_va_decoder_add_slice_buffer (base->decoder, va_pic, &slice_param,
357 sizeof (slice_param), (gpointer) picture->data, picture->size);
361 gst_va_vp8_dec_decode_picture (GstVp8Decoder * decoder, GstVp8Picture * picture,
362 GstVp8Parser * parser)
364 if (_fill_picture (decoder, picture, parser) &&
365 _add_slice (decoder, picture, parser))
368 return GST_FLOW_ERROR;
372 gst_va_vp8_dec_end_picture (GstVp8Decoder * decoder, GstVp8Picture * picture)
374 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
375 GstVaDecodePicture *va_pic;
377 GST_LOG_OBJECT (base, "end picture %p, (system_frame_number %d)",
378 picture, picture->system_frame_number);
380 va_pic = gst_vp8_picture_get_user_data (picture);
382 if (!gst_va_decoder_decode (base->decoder, va_pic))
383 return GST_FLOW_ERROR;
389 gst_va_vp8_dec_output_picture (GstVp8Decoder * decoder,
390 GstVideoCodecFrame * frame, GstVp8Picture * picture)
392 GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
393 GstVaVp8Dec *self = GST_VA_VP8_DEC (decoder);
394 GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
397 GST_LOG_OBJECT (self,
398 "Outputting picture %p (system_frame_number %d)",
399 picture, picture->system_frame_number);
401 ret = gst_va_base_dec_process_output (base, frame, picture->discont_state, 0);
402 gst_vp8_picture_unref (picture);
405 return gst_video_decoder_finish_frame (vdec, frame);
406 return GST_FLOW_ERROR;
410 gst_va_vp8_dec_init (GTypeInstance * instance, gpointer g_class)
412 gst_va_base_dec_init (GST_VA_BASE_DEC (instance), GST_CAT_DEFAULT);
416 gst_va_vp8_dec_dispose (GObject * object)
418 gst_va_base_dec_close (GST_VIDEO_DECODER (object));
419 G_OBJECT_CLASS (parent_class)->dispose (object);
423 gst_va_vp8_dec_class_init (gpointer g_class, gpointer class_data)
425 GstCaps *src_doc_caps, *sink_doc_caps;
426 GObjectClass *gobject_class = G_OBJECT_CLASS (g_class);
427 GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
428 GstVp8DecoderClass *vp8decoder_class = GST_VP8_DECODER_CLASS (g_class);
429 struct CData *cdata = class_data;
432 if (cdata->description) {
433 long_name = g_strdup_printf ("VA-API VP8 Decoder in %s",
436 long_name = g_strdup ("VA-API VP8 Decoder");
439 gst_element_class_set_metadata (element_class, long_name,
440 "Codec/Decoder/Video/Hardware",
441 "VA-API based VP8 video decoder", "He Junyan <junyan.he@intel.com>");
443 sink_doc_caps = gst_caps_from_string (sink_caps_str);
444 src_doc_caps = gst_caps_from_string (src_caps_str);
446 parent_class = g_type_class_peek_parent (g_class);
449 * GstVaVp8Dec:device-path:
451 * It shows the DRM device path used for the VA operation, if any.
455 gst_va_base_dec_class_init (GST_VA_BASE_DEC_CLASS (g_class), VP8,
456 cdata->render_device_path, cdata->sink_caps, cdata->src_caps,
457 src_doc_caps, sink_doc_caps);
459 gobject_class->dispose = gst_va_vp8_dec_dispose;
461 vp8decoder_class->new_sequence =
462 GST_DEBUG_FUNCPTR (gst_va_vp8_dec_new_sequence);
463 vp8decoder_class->new_picture =
464 GST_DEBUG_FUNCPTR (gst_va_vp8_dec_new_picture);
465 vp8decoder_class->decode_picture =
466 GST_DEBUG_FUNCPTR (gst_va_vp8_dec_decode_picture);
467 vp8decoder_class->end_picture =
468 GST_DEBUG_FUNCPTR (gst_va_vp8_dec_end_picture);
469 vp8decoder_class->output_picture =
470 GST_DEBUG_FUNCPTR (gst_va_vp8_dec_output_picture);
473 g_free (cdata->description);
474 g_free (cdata->render_device_path);
475 gst_caps_unref (cdata->src_caps);
476 gst_caps_unref (cdata->sink_caps);
481 _register_debug_category (gpointer data)
483 GST_DEBUG_CATEGORY_INIT (gst_va_vp8dec_debug, "vavp8dec", 0,
490 gst_va_vp8_dec_register (GstPlugin * plugin, GstVaDevice * device,
491 GstCaps * sink_caps, GstCaps * src_caps, guint rank)
493 static GOnce debug_once = G_ONCE_INIT;
495 GTypeInfo type_info = {
496 .class_size = sizeof (GstVaVp8DecClass),
497 .class_init = gst_va_vp8_dec_class_init,
498 .instance_size = sizeof (GstVaVp8Dec),
499 .instance_init = gst_va_vp8_dec_init,
503 gchar *type_name, *feature_name;
505 g_return_val_if_fail (GST_IS_PLUGIN (plugin), FALSE);
506 g_return_val_if_fail (GST_IS_VA_DEVICE (device), FALSE);
507 g_return_val_if_fail (GST_IS_CAPS (sink_caps), FALSE);
508 g_return_val_if_fail (GST_IS_CAPS (src_caps), FALSE);
510 cdata = g_new (struct CData, 1);
511 cdata->description = NULL;
512 cdata->render_device_path = g_strdup (device->render_device_path);
513 cdata->sink_caps = gst_caps_ref (sink_caps);
514 cdata->src_caps = gst_caps_ref (src_caps);
516 /* class data will be leaked if the element never gets instantiated */
517 GST_MINI_OBJECT_FLAG_SET (cdata->sink_caps,
518 GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
519 GST_MINI_OBJECT_FLAG_SET (src_caps, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
521 type_info.class_data = cdata;
523 /* The first decoder to be registered should use a constant name,
524 * like vavp8dec, for any additional decoders, we create unique
525 * names, using inserting the render device name. */
526 if (device->index == 0) {
527 type_name = g_strdup ("GstVaVp8Dec");
528 feature_name = g_strdup ("vavp8dec");
530 gchar *basename = g_path_get_basename (device->render_device_path);
531 type_name = g_strdup_printf ("GstVa%sVp8Dec", basename);
532 feature_name = g_strdup_printf ("va%svp8dec", basename);
533 cdata->description = basename;
535 /* lower rank for non-first device */
540 g_once (&debug_once, _register_debug_category, NULL);
542 type = g_type_register_static (GST_TYPE_VP8_DECODER,
543 type_name, &type_info, 0);
545 ret = gst_element_register (plugin, feature_name, rank, type);
548 g_free (feature_name);