2 * gstvaapidecode.c - VA-API video decoder
4 * Copyright (C) 2010-2011 Splitted-Desktop Systems
5 * Copyright (C) 2011-2012 Intel Corporation
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapidecode
25 * @short_description: A VA-API based video decoder
27 * vaapidecode decodes from raw bitstreams to surfaces suitable for
28 * the vaapisink element.
33 #include <gst/vaapi/gstvaapidisplay.h>
34 #include <gst/vaapi/gstvaapivideosink.h>
35 #include <gst/vaapi/gstvaapivideobuffer.h>
36 #include <gst/video/videocontext.h>
39 #include <gst/vaapi/gstvaapivideobuffer_glx.h>
40 #define gst_vaapi_video_buffer_new(display) \
41 gst_vaapi_video_buffer_glx_new(GST_VAAPI_DISPLAY_GLX(display))
44 #include "gstvaapidecode.h"
45 #include "gstvaapipluginutil.h"
48 # include <gst/vaapi/gstvaapidecoder_ffmpeg.h>
51 # include <gst/vaapi/gstvaapidecoder_h264.h>
52 # include <gst/vaapi/gstvaapidecoder_jpeg.h>
53 # include <gst/vaapi/gstvaapidecoder_mpeg2.h>
54 # include <gst/vaapi/gstvaapidecoder_mpeg4.h>
55 # include <gst/vaapi/gstvaapidecoder_vc1.h>
58 /* Favor codecparsers-based decoders for 0.3.x series */
59 #define USE_FFMPEG_DEFAULT \
60 (USE_FFMPEG && !USE_CODEC_PARSERS)
62 #define GST_PLUGIN_NAME "vaapidecode"
63 #define GST_PLUGIN_DESC "A VA-API based video decoder"
65 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidecode);
66 #define GST_CAT_DEFAULT gst_debug_vaapidecode
68 /* ElementFactory information */
69 static const GstElementDetails gst_vaapidecode_details =
72 "Codec/Decoder/Video",
74 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
76 /* Default templates */
77 #define GST_CAPS_CODEC(CODEC) CODEC "; "
79 static const char gst_vaapidecode_sink_caps_str[] =
80 GST_CAPS_CODEC("video/mpeg, mpegversion=2, systemstream=(boolean)false")
81 GST_CAPS_CODEC("video/mpeg, mpegversion=4")
82 GST_CAPS_CODEC("video/x-divx")
83 GST_CAPS_CODEC("video/x-xvid")
84 GST_CAPS_CODEC("video/x-h263")
85 GST_CAPS_CODEC("video/x-h264")
86 GST_CAPS_CODEC("video/x-wmv")
87 GST_CAPS_CODEC("image/jpeg")
90 static const char gst_vaapidecode_src_caps_str[] =
91 GST_VAAPI_SURFACE_CAPS;
93 static GstStaticPadTemplate gst_vaapidecode_sink_factory =
94 GST_STATIC_PAD_TEMPLATE(
98 GST_STATIC_CAPS(gst_vaapidecode_sink_caps_str));
100 static GstStaticPadTemplate gst_vaapidecode_src_factory =
101 GST_STATIC_PAD_TEMPLATE(
105 GST_STATIC_CAPS(gst_vaapidecode_src_caps_str));
107 #define GstVideoContextClass GstVideoContextInterface
108 GST_BOILERPLATE_WITH_INTERFACE(
114 GST_TYPE_VIDEO_CONTEXT,
124 gst_vaapidecode_update_src_caps(GstVaapiDecode *decode, GstCaps *caps);
127 gst_vaapi_decoder_notify_caps(GObject *obj, GParamSpec *pspec, void *user_data)
129 GstVaapiDecode * const decode = GST_VAAPIDECODE(user_data);
132 g_assert(decode->decoder == GST_VAAPI_DECODER(obj));
134 caps = gst_vaapi_decoder_get_caps(decode->decoder);
135 gst_vaapidecode_update_src_caps(decode, caps);
138 static inline gboolean
139 gst_vaapidecode_update_sink_caps(GstVaapiDecode *decode, GstCaps *caps)
141 if (decode->sinkpad_caps)
142 gst_caps_unref(decode->sinkpad_caps);
143 decode->sinkpad_caps = gst_caps_ref(caps);
148 gst_vaapidecode_update_src_caps(GstVaapiDecode *decode, GstCaps *caps)
151 GstStructure *structure;
152 const GValue *v_width, *v_height, *v_framerate, *v_par, *v_interlaced;
155 if (!decode->srcpad_caps) {
156 decode->srcpad_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS_NAME);
157 if (!decode->srcpad_caps)
161 structure = gst_caps_get_structure(caps, 0);
162 v_width = gst_structure_get_value(structure, "width");
163 v_height = gst_structure_get_value(structure, "height");
164 v_framerate = gst_structure_get_value(structure, "framerate");
165 v_par = gst_structure_get_value(structure, "pixel-aspect-ratio");
166 v_interlaced = gst_structure_get_value(structure, "interlaced");
168 structure = gst_caps_get_structure(decode->srcpad_caps, 0);
169 if (v_width && v_height) {
170 gst_structure_set_value(structure, "width", v_width);
171 gst_structure_set_value(structure, "height", v_height);
174 gst_structure_set_value(structure, "framerate", v_framerate);
176 gst_structure_set_value(structure, "pixel-aspect-ratio", v_par);
178 gst_structure_set_value(structure, "interlaced", v_interlaced);
180 gst_structure_set(structure, "type", G_TYPE_STRING, "vaapi", NULL);
181 gst_structure_set(structure, "opengl", G_TYPE_BOOLEAN, USE_VAAPI_GLX, NULL);
183 other_caps = gst_caps_copy(decode->srcpad_caps);
184 success = gst_pad_set_caps(decode->srcpad, other_caps);
185 gst_caps_unref(other_caps);
190 gst_vaapidecode_release(GstVaapiDecode *decode, GObject *dead_object)
192 g_mutex_lock(decode->decoder_mutex);
193 g_cond_signal(decode->decoder_ready);
194 g_mutex_unlock(decode->decoder_mutex);
198 gst_vaapidecode_step(GstVaapiDecode *decode)
200 GstVaapiSurfaceProxy *proxy;
201 GstVaapiDecoderStatus status;
209 proxy = gst_vaapi_decoder_get_surface(decode->decoder, &status);
211 if (status == GST_VAAPI_DECODER_STATUS_ERROR_NO_SURFACE) {
212 /* Wait for a VA surface to be displayed and free'd */
214 goto error_decode_timeout;
216 g_get_current_time(&timeout);
217 g_time_val_add(&timeout, 10000); /* 10 ms each step */
218 g_mutex_lock(decode->decoder_mutex);
220 decode->decoder_ready,
221 decode->decoder_mutex,
224 g_mutex_unlock(decode->decoder_mutex);
227 if (status != GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA)
229 /* More data is needed */
235 (GWeakNotify)gst_vaapidecode_release,
239 buffer = gst_vaapi_video_buffer_new(decode->display);
241 goto error_create_buffer;
243 GST_BUFFER_TIMESTAMP(buffer) = GST_VAAPI_SURFACE_PROXY_TIMESTAMP(proxy);
244 gst_buffer_set_caps(buffer, GST_PAD_CAPS(decode->srcpad));
246 if (GST_VAAPI_SURFACE_PROXY_TFF(proxy))
247 GST_BUFFER_FLAG_SET(buffer, GST_VIDEO_BUFFER_TFF);
249 gst_vaapi_video_buffer_set_surface_proxy(
250 GST_VAAPI_VIDEO_BUFFER(buffer),
254 ret = gst_pad_push(decode->srcpad, buffer);
255 if (ret != GST_FLOW_OK)
256 goto error_commit_buffer;
258 g_object_unref(proxy);
263 error_decode_timeout:
265 GST_DEBUG("decode timeout. Decoder required a VA surface but none "
266 "got available within one second");
267 return GST_FLOW_UNEXPECTED;
271 GST_DEBUG("decode error %d", status);
273 case GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC:
274 case GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE:
275 case GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT:
276 ret = GST_FLOW_NOT_SUPPORTED;
279 ret = GST_FLOW_UNEXPECTED;
286 const GstVaapiID surface_id =
287 gst_vaapi_surface_get_id(GST_VAAPI_SURFACE_PROXY_SURFACE(proxy));
289 GST_DEBUG("video sink failed to create video buffer for proxy'ed "
290 "surface %" GST_VAAPI_ID_FORMAT " (error %d)",
291 GST_VAAPI_ID_ARGS(surface_id), ret);
292 g_object_unref(proxy);
293 return GST_FLOW_UNEXPECTED;
297 GST_DEBUG("video sink rejected the video buffer (error %d)", ret);
298 g_object_unref(proxy);
299 return GST_FLOW_UNEXPECTED;
304 gst_vaapidecode_create(GstVaapiDecode *decode, GstCaps *caps)
306 GstVaapiDisplay *dpy;
307 GstStructure *structure;
310 if (!gst_vaapi_ensure_display(decode, &decode->display))
313 decode->decoder_mutex = g_mutex_new();
314 if (!decode->decoder_mutex)
317 decode->decoder_ready = g_cond_new();
318 if (!decode->decoder_ready)
321 dpy = decode->display;
322 if (decode->use_ffmpeg) {
324 decode->decoder = gst_vaapi_decoder_ffmpeg_new(dpy, caps);
328 #if USE_CODEC_PARSERS
329 structure = gst_caps_get_structure(caps, 0);
332 if (gst_structure_has_name(structure, "video/x-h264"))
333 decode->decoder = gst_vaapi_decoder_h264_new(dpy, caps);
334 else if (gst_structure_has_name(structure, "video/mpeg")) {
335 if (!gst_structure_get_int(structure, "mpegversion", &version))
338 decode->decoder = gst_vaapi_decoder_mpeg2_new(dpy, caps);
339 else if (version == 4)
340 decode->decoder = gst_vaapi_decoder_mpeg4_new(dpy, caps);
342 else if (gst_structure_has_name(structure, "video/x-wmv"))
343 decode->decoder = gst_vaapi_decoder_vc1_new(dpy, caps);
344 else if (gst_structure_has_name(structure, "video/x-h263") ||
345 gst_structure_has_name(structure, "video/x-divx") ||
346 gst_structure_has_name(structure, "video/x-xvid"))
347 decode->decoder = gst_vaapi_decoder_mpeg4_new(dpy, caps);
349 else if (gst_structure_has_name(structure, "image/jpeg"))
350 decode->decoder = gst_vaapi_decoder_jpeg_new(dpy, caps);
354 if (!decode->decoder)
358 G_OBJECT(decode->decoder),
360 G_CALLBACK(gst_vaapi_decoder_notify_caps),
364 decode->decoder_caps = gst_caps_ref(caps);
369 gst_vaapidecode_destroy(GstVaapiDecode *decode)
371 if (decode->decoder) {
372 gst_vaapi_decoder_put_buffer(decode->decoder, NULL);
373 g_object_unref(decode->decoder);
374 decode->decoder = NULL;
377 if (decode->decoder_caps) {
378 gst_caps_unref(decode->decoder_caps);
379 decode->decoder_caps = NULL;
382 if (decode->decoder_ready) {
383 gst_vaapidecode_release(decode, NULL);
384 g_cond_free(decode->decoder_ready);
385 decode->decoder_ready = NULL;
388 if (decode->decoder_mutex) {
389 g_mutex_free(decode->decoder_mutex);
390 decode->decoder_mutex = NULL;
395 gst_vaapidecode_reset(GstVaapiDecode *decode, GstCaps *caps)
397 if (decode->decoder &&
398 decode->decoder_caps &&
399 gst_caps_is_always_compatible(caps, decode->decoder_caps))
402 gst_vaapidecode_destroy(decode);
403 return gst_vaapidecode_create(decode, caps);
406 /* GstVideoContext interface */
409 gst_vaapidecode_set_video_context(GstVideoContext *context, const gchar *type,
412 GstVaapiDecode *decode = GST_VAAPIDECODE (context);
413 gst_vaapi_set_display (type, value, &decode->display);
417 gst_video_context_supported (GstVaapiDecode *decode, GType iface_type)
419 return (iface_type == GST_TYPE_VIDEO_CONTEXT);
423 gst_video_context_interface_init(GstVideoContextInterface *iface)
425 iface->set_context = gst_vaapidecode_set_video_context;
429 gst_vaapidecode_base_init(gpointer klass)
431 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
432 GstPadTemplate *pad_template;
434 gst_element_class_set_details(element_class, &gst_vaapidecode_details);
437 pad_template = gst_static_pad_template_get(&gst_vaapidecode_sink_factory);
438 gst_element_class_add_pad_template(element_class, pad_template);
439 gst_object_unref(pad_template);
442 pad_template = gst_static_pad_template_get(&gst_vaapidecode_src_factory);
443 gst_element_class_add_pad_template(element_class, pad_template);
444 gst_object_unref(pad_template);
448 gst_vaapidecode_finalize(GObject *object)
450 GstVaapiDecode * const decode = GST_VAAPIDECODE(object);
452 gst_vaapidecode_destroy(decode);
454 if (decode->sinkpad_caps) {
455 gst_caps_unref(decode->sinkpad_caps);
456 decode->sinkpad_caps = NULL;
459 if (decode->srcpad_caps) {
460 gst_caps_unref(decode->srcpad_caps);
461 decode->srcpad_caps = NULL;
464 if (decode->display) {
465 g_object_unref(decode->display);
466 decode->display = NULL;
469 if (decode->allowed_caps) {
470 gst_caps_unref(decode->allowed_caps);
471 decode->allowed_caps = NULL;
474 if (decode->delayed_new_seg) {
475 gst_event_unref(decode->delayed_new_seg);
476 decode->delayed_new_seg = NULL;
479 G_OBJECT_CLASS(parent_class)->finalize(object);
483 gst_vaapidecode_set_property(
490 GstVaapiDecode * const decode = GST_VAAPIDECODE(object);
493 case PROP_USE_FFMPEG:
494 decode->use_ffmpeg = g_value_get_boolean(value);
497 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
503 gst_vaapidecode_get_property(
510 GstVaapiDecode * const decode = GST_VAAPIDECODE(object);
513 case PROP_USE_FFMPEG:
514 g_value_set_boolean(value, decode->use_ffmpeg);
517 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
522 static GstStateChangeReturn
523 gst_vaapidecode_change_state(GstElement *element, GstStateChange transition)
525 GstVaapiDecode * const decode = GST_VAAPIDECODE(element);
526 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
528 switch (transition) {
529 case GST_STATE_CHANGE_NULL_TO_READY:
530 decode->is_ready = TRUE;
532 case GST_STATE_CHANGE_READY_TO_PAUSED:
534 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
540 ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
541 if (ret != GST_STATE_CHANGE_SUCCESS)
544 switch (transition) {
545 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
547 case GST_STATE_CHANGE_PAUSED_TO_READY:
549 case GST_STATE_CHANGE_READY_TO_NULL:
550 gst_vaapidecode_destroy(decode);
551 if (decode->display) {
552 g_object_unref(decode->display);
553 decode->display = NULL;
555 decode->is_ready = FALSE;
560 return GST_STATE_CHANGE_SUCCESS;
564 gst_vaapidecode_class_init(GstVaapiDecodeClass *klass)
566 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
567 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
569 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidecode,
570 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
572 object_class->finalize = gst_vaapidecode_finalize;
573 object_class->set_property = gst_vaapidecode_set_property;
574 object_class->get_property = gst_vaapidecode_get_property;
576 element_class->change_state = gst_vaapidecode_change_state;
579 g_object_class_install_property
582 g_param_spec_boolean("use-ffmpeg",
583 "Use FFmpeg/VAAPI for decoding",
584 "Uses FFmpeg/VAAPI for decoding",
591 gst_vaapidecode_ensure_allowed_caps(GstVaapiDecode *decode)
593 GstCaps *decode_caps;
594 guint i, n_decode_caps;
596 if (decode->allowed_caps)
599 if (!gst_vaapi_ensure_display(decode, &decode->display))
600 goto error_no_display;
602 decode_caps = gst_vaapi_display_get_decode_caps(decode->display);
604 goto error_no_decode_caps;
605 n_decode_caps = gst_caps_get_size(decode_caps);
607 decode->allowed_caps = gst_caps_new_empty();
608 if (!decode->allowed_caps)
609 goto error_no_memory;
611 for (i = 0; i < n_decode_caps; i++) {
612 GstStructure *structure;
613 structure = gst_caps_get_structure(decode_caps, i);
616 structure = gst_structure_copy(structure);
619 gst_structure_remove_field(structure, "profile");
622 "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
623 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
626 gst_caps_merge_structure(decode->allowed_caps, structure);
629 gst_caps_unref(decode_caps);
635 GST_DEBUG("failed to retrieve VA display");
638 error_no_decode_caps:
640 GST_DEBUG("failed to retrieve VA decode caps");
645 GST_DEBUG("failed to allocate allowed-caps set");
646 gst_caps_unref(decode_caps);
652 gst_vaapidecode_get_caps(GstPad *pad)
654 GstVaapiDecode * const decode = GST_VAAPIDECODE(GST_OBJECT_PARENT(pad));
656 if (!decode->is_ready)
657 return gst_static_pad_template_get_caps(&gst_vaapidecode_sink_factory);
659 if (!gst_vaapidecode_ensure_allowed_caps(decode))
660 return gst_caps_new_empty();
662 return gst_caps_ref(decode->allowed_caps);
666 gst_vaapidecode_set_caps(GstPad *pad, GstCaps *caps)
668 GstVaapiDecode * const decode = GST_VAAPIDECODE(GST_OBJECT_PARENT(pad));
670 g_return_val_if_fail(pad == decode->sinkpad, FALSE);
672 if (!gst_vaapidecode_update_sink_caps(decode, caps))
674 if (!gst_vaapidecode_update_src_caps(decode, caps))
676 if (!gst_vaapidecode_reset(decode, decode->sinkpad_caps))
679 /* Propagate NEWSEGMENT event downstream, now that pads are linked */
680 if (decode->delayed_new_seg) {
681 if (gst_pad_push_event(decode->srcpad, decode->delayed_new_seg))
682 gst_event_unref(decode->delayed_new_seg);
683 decode->delayed_new_seg = NULL;
689 gst_vaapidecode_chain(GstPad *pad, GstBuffer *buf)
691 GstVaapiDecode * const decode = GST_VAAPIDECODE(GST_OBJECT_PARENT(pad));
693 if (!gst_vaapi_decoder_put_buffer(decode->decoder, buf))
694 goto error_push_buffer;
696 gst_buffer_unref(buf);
697 return gst_vaapidecode_step(decode);
702 GST_DEBUG("failed to push input buffer to decoder");
703 gst_buffer_unref(buf);
704 return GST_FLOW_UNEXPECTED;
709 gst_vaapidecode_sink_event(GstPad *pad, GstEvent *event)
711 GstVaapiDecode * const decode = GST_VAAPIDECODE(GST_OBJECT_PARENT(pad));
713 GST_DEBUG("handle sink event '%s'", GST_EVENT_TYPE_NAME(event));
715 /* Propagate event downstream */
716 switch (GST_EVENT_TYPE(event)) {
717 case GST_EVENT_NEWSEGMENT:
718 if (decode->delayed_new_seg) {
719 gst_event_unref(decode->delayed_new_seg);
720 decode->delayed_new_seg = NULL;
722 if (!GST_PAD_PEER(decode->srcpad)) {
723 decode->delayed_new_seg = gst_event_ref(event);
730 return gst_pad_push_event(decode->srcpad, event);
734 gst_vaapidecode_src_event(GstPad *pad, GstEvent *event)
736 GstVaapiDecode * const decode = GST_VAAPIDECODE(GST_OBJECT_PARENT(pad));
738 GST_DEBUG("handle src event '%s'", GST_EVENT_TYPE_NAME(event));
740 /* Propagate event upstream */
741 return gst_pad_push_event(decode->sinkpad, event);
745 gst_vaapidecode_query (GstPad *pad, GstQuery *query) {
746 GstVaapiDecode *decode = GST_VAAPIDECODE (gst_pad_get_parent_element (pad));
749 GST_DEBUG ("sharing display %p", decode->display);
751 if (gst_vaapi_reply_to_query (query, decode->display))
754 res = gst_pad_query_default (pad, query);
756 g_object_unref (decode);
761 gst_vaapidecode_init(GstVaapiDecode *decode, GstVaapiDecodeClass *klass)
763 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
765 decode->display = NULL;
766 decode->decoder = NULL;
767 decode->decoder_mutex = NULL;
768 decode->decoder_ready = NULL;
769 decode->decoder_caps = NULL;
770 decode->allowed_caps = NULL;
771 decode->delayed_new_seg = NULL;
772 decode->use_ffmpeg = USE_FFMPEG_DEFAULT;
773 decode->is_ready = FALSE;
775 /* Pad through which data comes in to the element */
776 decode->sinkpad = gst_pad_new_from_template(
777 gst_element_class_get_pad_template(element_class, "sink"),
780 decode->sinkpad_caps = NULL;
782 gst_pad_set_getcaps_function(decode->sinkpad, gst_vaapidecode_get_caps);
783 gst_pad_set_setcaps_function(decode->sinkpad, gst_vaapidecode_set_caps);
784 gst_pad_set_chain_function(decode->sinkpad, gst_vaapidecode_chain);
785 gst_pad_set_event_function(decode->sinkpad, gst_vaapidecode_sink_event);
786 gst_pad_set_query_function(decode->sinkpad, gst_vaapidecode_query);
787 gst_element_add_pad(GST_ELEMENT(decode), decode->sinkpad);
789 /* Pad through which data goes out of the element */
790 decode->srcpad = gst_pad_new_from_template(
791 gst_element_class_get_pad_template(element_class, "src"),
794 decode->srcpad_caps = NULL;
796 gst_pad_use_fixed_caps(decode->srcpad);
797 gst_pad_set_event_function(decode->srcpad, gst_vaapidecode_src_event);
798 gst_pad_set_query_function(decode->srcpad, gst_vaapidecode_query);
799 gst_element_add_pad(GST_ELEMENT(decode), decode->srcpad);