2 * gstvaapipostproc.c - VA-API video postprocessing
4 * Copyright (C) 2012-2013 Intel Corporation
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either version 2.1
9 * of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301 USA
23 * SECTION:gstvaapipostproc
24 * @short_description: A video postprocessing filter
26 * vaapipostproc consists in various postprocessing algorithms to be
27 * applied to VA surfaces. So far, only basic bob deinterlacing is
31 #include "gst/vaapi/sysdeps.h"
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
35 #include "gstvaapipostproc.h"
36 #include "gstvaapipluginutil.h"
37 #include "gstvaapivideobuffer.h"
39 #define GST_PLUGIN_NAME "vaapipostproc"
40 #define GST_PLUGIN_DESC "A video postprocessing filter"
42 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapipostproc);
43 #define GST_CAT_DEFAULT gst_debug_vaapipostproc
45 /* Default templates */
46 static const char gst_vaapipostproc_sink_caps_str[] =
47 GST_VAAPI_SURFACE_CAPS ", "
48 GST_CAPS_INTERLACED_MODES;
50 static const char gst_vaapipostproc_src_caps_str[] =
51 GST_VAAPI_SURFACE_CAPS ", "
52 GST_CAPS_INTERLACED_FALSE;
54 static GstStaticPadTemplate gst_vaapipostproc_sink_factory =
55 GST_STATIC_PAD_TEMPLATE(
59 GST_STATIC_CAPS(gst_vaapipostproc_sink_caps_str));
61 static GstStaticPadTemplate gst_vaapipostproc_src_factory =
62 GST_STATIC_PAD_TEMPLATE(
66 GST_STATIC_CAPS(gst_vaapipostproc_src_caps_str));
68 /* GstImplementsInterface interface */
69 #if !GST_CHECK_VERSION(1,0,0)
71 gst_vaapipostproc_implements_interface_supported(
72 GstImplementsInterface *iface,
76 return (type == GST_TYPE_VIDEO_CONTEXT);
80 gst_vaapipostproc_implements_iface_init(GstImplementsInterfaceClass *iface)
82 iface->supported = gst_vaapipostproc_implements_interface_supported;
86 /* GstVideoContext interface */
88 gst_vaapipostproc_set_video_context(
89 GstVideoContext *context,
94 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(context);
96 gst_vaapi_set_display(type, value, &postproc->display);
100 gst_video_context_interface_init(GstVideoContextInterface *iface)
102 iface->set_context = gst_vaapipostproc_set_video_context;
105 #define GstVideoContextClass GstVideoContextInterface
106 G_DEFINE_TYPE_WITH_CODE(
110 #if !GST_CHECK_VERSION(1,0,0)
111 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
112 gst_vaapipostproc_implements_iface_init);
114 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
115 gst_video_context_interface_init))
120 PROP_DEINTERLACE_MODE,
121 PROP_DEINTERLACE_METHOD,
124 #define DEFAULT_DEINTERLACE_MODE GST_VAAPI_DEINTERLACE_MODE_AUTO
125 #define DEFAULT_DEINTERLACE_METHOD GST_VAAPI_DEINTERLACE_METHOD_BOB
127 #define GST_VAAPI_TYPE_DEINTERLACE_MODE \
128 gst_vaapi_deinterlace_mode_get_type()
131 gst_vaapi_deinterlace_mode_get_type(void)
133 static GType deinterlace_mode_type = 0;
135 static const GEnumValue mode_types[] = {
136 { GST_VAAPI_DEINTERLACE_MODE_AUTO,
137 "Auto detection", "auto" },
138 { GST_VAAPI_DEINTERLACE_MODE_INTERLACED,
139 "Force deinterlacing", "interlaced" },
140 { GST_VAAPI_DEINTERLACE_MODE_DISABLED,
141 "Never deinterlace", "disabled" },
145 if (!deinterlace_mode_type) {
146 deinterlace_mode_type =
147 g_enum_register_static("GstVaapiDeinterlaceMode", mode_types);
149 return deinterlace_mode_type;
152 static inline GstVaapiPostproc *
153 get_vaapipostproc_from_pad(GstPad *pad)
155 return GST_VAAPIPOSTPROC(gst_pad_get_parent_element(pad));
158 static inline gboolean
159 gst_vaapipostproc_ensure_display(GstVaapiPostproc *postproc)
161 return gst_vaapi_ensure_display(postproc, GST_VAAPI_DISPLAY_TYPE_ANY,
166 gst_vaapipostproc_create(GstVaapiPostproc *postproc, GstCaps *caps)
168 if (!gst_vaapipostproc_ensure_display(postproc))
171 gst_caps_replace(&postproc->postproc_caps, caps);
176 gst_vaapipostproc_destroy(GstVaapiPostproc *postproc)
178 gst_caps_replace(&postproc->postproc_caps, NULL);
180 gst_vaapi_display_replace(&postproc->display, NULL);
184 gst_vaapipostproc_reset(GstVaapiPostproc *postproc, GstCaps *caps)
186 if (postproc->postproc_caps &&
187 gst_caps_is_always_compatible(caps, postproc->postproc_caps))
190 gst_vaapipostproc_destroy(postproc);
191 return gst_vaapipostproc_create(postproc, caps);
195 gst_vaapipostproc_start(GstVaapiPostproc *postproc)
197 if (!gst_vaapipostproc_ensure_display(postproc))
203 gst_vaapipostproc_stop(GstVaapiPostproc *postproc)
205 gst_vaapi_display_replace(&postproc->display, NULL);
210 gst_vaapipostproc_process(GstVaapiPostproc *postproc, GstBuffer *buf)
212 GstVaapiVideoMeta *meta;
213 GstVaapiSurfaceProxy *proxy;
214 GstClockTime timestamp;
217 guint outbuf_flags, flags;
220 meta = gst_buffer_get_vaapi_video_meta(buf);
222 goto error_invalid_buffer;
224 /* Deinterlacing disabled, push frame */
225 if (!postproc->deinterlace) {
226 outbuf = gst_buffer_ref(buf);
228 goto error_create_buffer;
229 ret = gst_pad_push(postproc->srcpad, outbuf);
230 if (ret != GST_FLOW_OK)
231 goto error_push_buffer;
232 gst_buffer_unref(buf);
236 timestamp = GST_BUFFER_TIMESTAMP(buf);
237 proxy = gst_vaapi_video_meta_get_surface_proxy(meta);
238 tff = GST_BUFFER_FLAG_IS_SET(buf, GST_VIDEO_BUFFER_FLAG_TFF);
240 flags = gst_vaapi_video_meta_get_render_flags(meta) &
241 ~(GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD|
242 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD);
245 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
247 goto error_create_buffer;
249 meta = gst_buffer_get_vaapi_video_meta(outbuf);
250 outbuf_flags = flags;
251 outbuf_flags |= postproc->deinterlace ? (
253 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD :
254 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD) :
255 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
256 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
258 GST_BUFFER_TIMESTAMP(outbuf) = timestamp;
259 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
260 #if !GST_CHECK_VERSION(1,0,0)
261 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
263 ret = gst_pad_push(postproc->srcpad, outbuf);
264 if (ret != GST_FLOW_OK)
265 goto error_push_buffer;
268 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
270 goto error_create_buffer;
272 meta = gst_buffer_get_vaapi_video_meta(outbuf);
273 outbuf_flags = flags;
274 outbuf_flags |= postproc->deinterlace ? (
276 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD :
277 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD) :
278 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
279 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
281 GST_BUFFER_TIMESTAMP(outbuf) = timestamp + postproc->field_duration;
282 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
283 #if !GST_CHECK_VERSION(1,0,0)
284 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
286 ret = gst_pad_push(postproc->srcpad, outbuf);
287 if (ret != GST_FLOW_OK)
288 goto error_push_buffer;
290 gst_buffer_unref(buf);
294 error_invalid_buffer:
296 GST_ERROR("failed to receive a valid video buffer");
297 gst_buffer_unref(buf);
302 GST_ERROR("failed to create output buffer");
303 gst_buffer_unref(buf);
308 if (ret != GST_FLOW_FLUSHING)
309 GST_ERROR("failed to push output buffer to video sink");
310 gst_buffer_unref(buf);
316 gst_vaapipostproc_update_sink_caps(GstVaapiPostproc *postproc, GstCaps *caps)
320 if (!gst_video_info_from_caps(&vi, caps))
322 postproc->fps_n = GST_VIDEO_INFO_FPS_N(&vi);
323 postproc->fps_d = GST_VIDEO_INFO_FPS_D(&vi);
325 switch (postproc->deinterlace_mode) {
326 case GST_VAAPI_DEINTERLACE_MODE_AUTO:
327 postproc->deinterlace = GST_VIDEO_INFO_IS_INTERLACED(&vi);
329 case GST_VAAPI_DEINTERLACE_MODE_INTERLACED:
330 postproc->deinterlace = TRUE;
332 case GST_VAAPI_DEINTERLACE_MODE_DISABLED:
333 postproc->deinterlace = FALSE;
337 postproc->field_duration = gst_util_uint64_scale(
340 (1 + postproc->deinterlace) * postproc->fps_n
343 gst_caps_replace(&postproc->sinkpad_caps, caps);
348 gst_vaapipostproc_update_src_caps(GstVaapiPostproc *postproc, GstCaps *caps)
351 GstStructure *structure;
352 const GValue *v_width, *v_height, *v_par;
355 if (postproc->srcpad_caps)
356 src_caps = gst_caps_make_writable(postproc->srcpad_caps);
358 src_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS_NAME);
361 postproc->srcpad_caps = src_caps;
363 structure = gst_caps_get_structure(caps, 0);
364 v_width = gst_structure_get_value(structure, "width");
365 v_height = gst_structure_get_value(structure, "height");
366 v_par = gst_structure_get_value(structure, "pixel-aspect-ratio");
368 structure = gst_caps_get_structure(src_caps, 0);
369 if (v_width && v_height) {
370 gst_structure_set_value(structure, "width", v_width);
371 gst_structure_set_value(structure, "height", v_height);
374 gst_structure_set_value(structure, "pixel-aspect-ratio", v_par);
376 gst_structure_set(structure, "type", G_TYPE_STRING, "vaapi", NULL);
377 gst_structure_set(structure, "opengl", G_TYPE_BOOLEAN, USE_GLX, NULL);
379 if (!postproc->deinterlace)
380 gst_structure_remove_interlaced_field(structure);
382 /* Set double framerate in interlaced mode */
383 if (!gst_util_fraction_multiply(postproc->fps_n, postproc->fps_d,
388 gst_structure_set(structure, "framerate",
389 GST_TYPE_FRACTION, fps_n, fps_d, NULL);
390 gst_structure_set_interlaced(structure, FALSE);
392 return gst_pad_set_caps(postproc->srcpad, src_caps);
396 gst_vaapipostproc_ensure_allowed_caps(GstVaapiPostproc *postproc)
398 if (postproc->allowed_caps)
401 postproc->allowed_caps =
402 gst_caps_from_string(gst_vaapipostproc_sink_caps_str);
403 if (!postproc->allowed_caps)
406 /* XXX: append VA/VPP filters */
411 gst_vaapipostproc_get_caps(GstPad *pad)
413 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
416 if (gst_vaapipostproc_ensure_allowed_caps(postproc))
417 out_caps = gst_caps_ref(postproc->allowed_caps);
419 out_caps = gst_caps_new_empty();
421 gst_object_unref(postproc);
426 gst_vaapipostproc_set_caps(GstPad *pad, GstCaps *caps)
428 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
429 gboolean success = FALSE;
431 g_return_val_if_fail(pad == postproc->sinkpad, FALSE);
434 if (!gst_vaapipostproc_update_sink_caps(postproc, caps))
436 if (!gst_vaapipostproc_update_src_caps(postproc, caps))
438 if (!gst_vaapipostproc_reset(postproc, postproc->sinkpad_caps))
442 gst_object_unref(postproc);
447 gst_vaapipostproc_chain(GST_PAD_CHAIN_FUNCTION_ARGS)
449 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
452 ret = gst_vaapipostproc_process(postproc, buffer);
453 gst_object_unref(postproc);
458 gst_vaapipostproc_sink_event(GST_PAD_EVENT_FUNCTION_ARGS)
460 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
463 GST_DEBUG("handle sink event '%s'", GST_EVENT_TYPE_NAME(event));
465 switch (GST_EVENT_TYPE(event)) {
466 #if GST_CHECK_VERSION(1,0,0)
467 case GST_EVENT_CAPS: {
469 gst_event_parse_caps(event, &caps);
470 success = gst_vaapipostproc_set_caps(pad, caps);
475 /* Propagate event downstream */
476 success = gst_pad_push_event(postproc->srcpad, event);
479 gst_object_unref(postproc);
484 gst_vaapipostproc_src_event(GST_PAD_EVENT_FUNCTION_ARGS)
486 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
489 GST_DEBUG("handle src event '%s'", GST_EVENT_TYPE_NAME(event));
491 /* Propagate event upstream */
492 success = gst_pad_push_event(postproc->sinkpad, event);
493 gst_object_unref(postproc);
498 gst_vaapipostproc_query(GST_PAD_QUERY_FUNCTION_ARGS)
500 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
503 GST_DEBUG("sharing display %p", postproc->display);
505 if (gst_vaapi_reply_to_query(query, postproc->display))
507 #if GST_CHECK_VERSION(1,0,0)
508 else if (GST_PAD_IS_SINK(pad) && GST_QUERY_TYPE(query) == GST_QUERY_CAPS) {
509 GstCaps * const caps = gst_vaapipostproc_get_caps(pad);
510 gst_query_set_caps_result(query, caps);
511 gst_caps_unref(caps);
516 success = GST_PAD_QUERY_FUNCTION_CALL(gst_pad_query_default, pad,
519 gst_object_unref(postproc);
524 gst_vaapipostproc_finalize(GObject *object)
526 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
528 gst_vaapipostproc_destroy(postproc);
530 gst_caps_replace(&postproc->sinkpad_caps, NULL);
531 gst_caps_replace(&postproc->srcpad_caps, NULL);
532 gst_caps_replace(&postproc->allowed_caps, NULL);
534 G_OBJECT_CLASS(gst_vaapipostproc_parent_class)->finalize(object);
538 gst_vaapipostproc_set_property(
545 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
548 case PROP_DEINTERLACE_MODE:
549 postproc->deinterlace_mode = g_value_get_enum(value);
551 case PROP_DEINTERLACE_METHOD:
552 postproc->deinterlace_method = g_value_get_enum(value);
555 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
561 gst_vaapipostproc_get_property(
568 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
571 case PROP_DEINTERLACE_MODE:
572 g_value_set_enum(value, postproc->deinterlace_mode);
574 case PROP_DEINTERLACE_METHOD:
575 g_value_set_enum(value, postproc->deinterlace_method);
578 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
583 static GstStateChangeReturn
584 gst_vaapipostproc_change_state(GstElement *element, GstStateChange transition)
586 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(element);
587 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
589 switch (transition) {
590 case GST_STATE_CHANGE_NULL_TO_READY:
591 if (!gst_vaapipostproc_start(postproc))
592 return GST_STATE_CHANGE_FAILURE;
594 case GST_STATE_CHANGE_READY_TO_PAUSED:
596 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
602 ret = GST_ELEMENT_CLASS(gst_vaapipostproc_parent_class)->change_state(element, transition);
603 if (ret != GST_STATE_CHANGE_SUCCESS)
606 switch (transition) {
607 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
609 case GST_STATE_CHANGE_PAUSED_TO_READY:
611 case GST_STATE_CHANGE_READY_TO_NULL:
612 if (!gst_vaapipostproc_stop(postproc))
613 return GST_STATE_CHANGE_FAILURE;
618 return GST_STATE_CHANGE_SUCCESS;
622 gst_vaapipostproc_class_init(GstVaapiPostprocClass *klass)
624 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
625 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
626 GstPadTemplate *pad_template;
628 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapipostproc,
629 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
631 object_class->finalize = gst_vaapipostproc_finalize;
632 object_class->set_property = gst_vaapipostproc_set_property;
633 object_class->get_property = gst_vaapipostproc_get_property;
635 element_class->change_state = gst_vaapipostproc_change_state;
637 gst_element_class_set_static_metadata(element_class,
638 "VA-API video postprocessing",
639 "Filter/Converter/Video",
641 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
644 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_sink_factory);
645 gst_element_class_add_pad_template(element_class, pad_template);
648 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_src_factory);
649 gst_element_class_add_pad_template(element_class, pad_template);
652 * GstVaapiPostproc:deinterlace-mode:
654 * This selects whether the deinterlacing should always be applied or if
655 * they should only be applied on content that has the "interlaced" flag
658 g_object_class_install_property
660 PROP_DEINTERLACE_MODE,
661 g_param_spec_enum("deinterlace",
663 "Deinterlace mode to use",
664 GST_VAAPI_TYPE_DEINTERLACE_MODE,
665 DEFAULT_DEINTERLACE_MODE,
666 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
669 * GstVaapiPostproc:deinterlace-method:
671 * This selects the deinterlacing method to apply.
673 g_object_class_install_property
675 PROP_DEINTERLACE_METHOD,
676 g_param_spec_enum("deinterlace-method",
677 "Deinterlace method",
678 "Deinterlace method to use",
679 GST_VAAPI_TYPE_DEINTERLACE_METHOD,
680 DEFAULT_DEINTERLACE_METHOD,
681 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
685 gst_vaapipostproc_init(GstVaapiPostproc *postproc)
687 GstVaapiPostprocClass *klass = GST_VAAPIPOSTPROC_GET_CLASS(postproc);
688 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
690 postproc->allowed_caps = NULL;
691 postproc->postproc_caps = NULL;
692 postproc->display = NULL;
693 postproc->surface_width = 0;
694 postproc->surface_height = 0;
695 postproc->deinterlace = FALSE;
696 postproc->deinterlace_mode = DEFAULT_DEINTERLACE_MODE;
697 postproc->deinterlace_method = DEFAULT_DEINTERLACE_METHOD;
698 postproc->field_duration = GST_CLOCK_TIME_NONE;
702 /* Pad through which data comes in to the element */
703 postproc->sinkpad = gst_pad_new_from_template(
704 gst_element_class_get_pad_template(element_class, "sink"),
707 postproc->sinkpad_caps = NULL;
709 #if !GST_CHECK_VERSION(1,0,0)
710 gst_pad_set_getcaps_function(postproc->sinkpad, gst_vaapipostproc_get_caps);
711 gst_pad_set_setcaps_function(postproc->sinkpad, gst_vaapipostproc_set_caps);
713 gst_pad_set_chain_function(postproc->sinkpad, gst_vaapipostproc_chain);
714 gst_pad_set_event_function(postproc->sinkpad, gst_vaapipostproc_sink_event);
715 gst_pad_set_query_function(postproc->sinkpad, gst_vaapipostproc_query);
716 gst_element_add_pad(GST_ELEMENT(postproc), postproc->sinkpad);
718 /* Pad through which data goes out of the element */
719 postproc->srcpad = gst_pad_new_from_template(
720 gst_element_class_get_pad_template(element_class, "src"),
723 postproc->srcpad_caps = NULL;
725 gst_pad_set_event_function(postproc->srcpad, gst_vaapipostproc_src_event);
726 gst_pad_set_query_function(postproc->srcpad, gst_vaapipostproc_query);
727 gst_element_add_pad(GST_ELEMENT(postproc), postproc->srcpad);