2 * gstvaapipostproc.c - VA-API video postprocessing
4 * Copyright (C) 2012-2013 Intel Corporation
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either version 2.1
9 * of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301 USA
23 * SECTION:gstvaapipostproc
24 * @short_description: A video postprocessing filter
26 * vaapipostproc consists in various postprocessing algorithms to be
27 * applied to VA surfaces. So far, only basic bob deinterlacing is
31 #include "gst/vaapi/sysdeps.h"
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
34 #include <gst/vaapi/gstvaapivideometa.h>
36 #include "gstvaapipostproc.h"
37 #include "gstvaapipluginutil.h"
38 #include "gstvaapipluginbuffer.h"
40 #define GST_PLUGIN_NAME "vaapipostproc"
41 #define GST_PLUGIN_DESC "A video postprocessing filter"
43 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapipostproc);
44 #define GST_CAT_DEFAULT gst_debug_vaapipostproc
46 /* ElementFactory information */
47 static const GstElementDetails gst_vaapipostproc_details =
49 "VA-API video postprocessing",
50 "Filter/Converter/Video",
52 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
54 /* Default templates */
55 static const char gst_vaapipostproc_sink_caps_str[] =
56 GST_VAAPI_SURFACE_CAPS ", "
57 "interlaced = (boolean) { true, false }";
59 static const char gst_vaapipostproc_src_caps_str[] =
60 GST_VAAPI_SURFACE_CAPS ", "
61 "interlaced = (boolean) false";
63 static GstStaticPadTemplate gst_vaapipostproc_sink_factory =
64 GST_STATIC_PAD_TEMPLATE(
68 GST_STATIC_CAPS(gst_vaapipostproc_sink_caps_str));
70 static GstStaticPadTemplate gst_vaapipostproc_src_factory =
71 GST_STATIC_PAD_TEMPLATE(
75 GST_STATIC_CAPS(gst_vaapipostproc_src_caps_str));
77 /* GstImplementsInterface interface */
79 gst_vaapipostproc_implements_interface_supported(
80 GstImplementsInterface *iface,
84 return (type == GST_TYPE_VIDEO_CONTEXT);
88 gst_vaapipostproc_implements_iface_init(GstImplementsInterfaceClass *iface)
90 iface->supported = gst_vaapipostproc_implements_interface_supported;
93 /* GstVideoContext interface */
95 gst_vaapipostproc_set_video_context(
96 GstVideoContext *context,
101 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(context);
103 gst_vaapi_set_display(type, value, &postproc->display);
107 gst_video_context_interface_init(GstVideoContextInterface *iface)
109 iface->set_context = gst_vaapipostproc_set_video_context;
112 #define GstVideoContextClass GstVideoContextInterface
113 G_DEFINE_TYPE_WITH_CODE(
117 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
118 gst_vaapipostproc_implements_iface_init);
119 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
120 gst_video_context_interface_init))
125 PROP_DEINTERLACE_MODE,
126 PROP_DEINTERLACE_METHOD,
129 #define DEFAULT_DEINTERLACE_MODE GST_VAAPI_DEINTERLACE_MODE_AUTO
130 #define DEFAULT_DEINTERLACE_METHOD GST_VAAPI_DEINTERLACE_METHOD_BOB
132 #define GST_VAAPI_TYPE_DEINTERLACE_MODE \
133 gst_vaapi_deinterlace_mode_get_type()
136 gst_vaapi_deinterlace_mode_get_type(void)
138 static GType deinterlace_mode_type = 0;
140 static const GEnumValue mode_types[] = {
141 { GST_VAAPI_DEINTERLACE_MODE_AUTO,
142 "Auto detection", "auto" },
143 { GST_VAAPI_DEINTERLACE_MODE_INTERLACED,
144 "Force deinterlacing", "interlaced" },
145 { GST_VAAPI_DEINTERLACE_MODE_DISABLED,
146 "Never deinterlace", "disabled" },
150 if (!deinterlace_mode_type) {
151 deinterlace_mode_type =
152 g_enum_register_static("GstVaapiDeinterlaceMode", mode_types);
154 return deinterlace_mode_type;
157 #define GST_VAAPI_TYPE_DEINTERLACE_METHOD \
158 gst_vaapi_deinterlace_method_get_type()
161 gst_vaapi_deinterlace_method_get_type(void)
163 static GType deinterlace_method_type = 0;
165 static const GEnumValue method_types[] = {
166 { GST_VAAPI_DEINTERLACE_METHOD_BOB,
167 "Bob deinterlacing", "bob" },
170 { GST_VAAPI_DEINTERLACE_METHOD_WEAVE,
171 "Weave deinterlacing", "weave" },
172 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_ADAPTIVE,
173 "Motion adaptive deinterlacing", "motion-adaptive" },
174 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_COMPENSATED,
175 "Motion compensated deinterlacing", "motion-compensated" },
180 if (!deinterlace_method_type) {
181 deinterlace_method_type =
182 g_enum_register_static("GstVaapiDeinterlaceMethod", method_types);
184 return deinterlace_method_type;
187 static inline GstVaapiPostproc *
188 get_vaapipostproc_from_pad(GstPad *pad)
190 return GST_VAAPIPOSTPROC(gst_pad_get_parent_element(pad));
193 static inline gboolean
194 gst_vaapipostproc_ensure_display(GstVaapiPostproc *postproc)
196 return gst_vaapi_ensure_display(postproc, GST_VAAPI_DISPLAY_TYPE_ANY,
201 gst_vaapipostproc_create(GstVaapiPostproc *postproc, GstCaps *caps)
203 if (!gst_vaapipostproc_ensure_display(postproc))
206 gst_caps_replace(&postproc->postproc_caps, caps);
211 gst_vaapipostproc_destroy(GstVaapiPostproc *postproc)
213 gst_caps_replace(&postproc->postproc_caps, NULL);
215 g_clear_object(&postproc->display);
219 gst_vaapipostproc_reset(GstVaapiPostproc *postproc, GstCaps *caps)
221 if (postproc->postproc_caps &&
222 gst_caps_is_always_compatible(caps, postproc->postproc_caps))
225 gst_vaapipostproc_destroy(postproc);
226 return gst_vaapipostproc_create(postproc, caps);
230 gst_vaapipostproc_start(GstVaapiPostproc *postproc)
232 if (!gst_vaapipostproc_ensure_display(postproc))
238 gst_vaapipostproc_stop(GstVaapiPostproc *postproc)
240 if (postproc->display) {
241 g_object_unref(postproc->display);
242 postproc->display = NULL;
248 gst_vaapipostproc_process(GstVaapiPostproc *postproc, GstBuffer *buf)
250 GstVaapiVideoMeta *meta;
251 GstVaapiSurfaceProxy *proxy;
252 GstClockTime timestamp;
254 GstBuffer *outbuf = NULL;
255 guint outbuf_flags, flags;
258 meta = gst_buffer_get_vaapi_video_meta(buf);
260 goto error_invalid_buffer;
262 flags = gst_vaapi_video_meta_get_render_flags(meta);
264 /* Deinterlacing disabled, push frame */
265 if (!postproc->deinterlace) {
266 gst_vaapi_video_meta_set_render_flags(meta, flags);
267 ret = gst_pad_push(postproc->srcpad, buf);
268 if (ret != GST_FLOW_OK)
269 goto error_push_buffer;
273 timestamp = GST_BUFFER_TIMESTAMP(buf);
274 proxy = gst_vaapi_video_meta_get_surface_proxy(meta);
275 tff = GST_BUFFER_FLAG_IS_SET(buf, GST_VIDEO_BUFFER_TFF);
277 flags &= ~(GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD|
278 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD);
281 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
283 goto error_create_buffer;
285 meta = gst_buffer_get_vaapi_video_meta(outbuf);
286 outbuf_flags = flags;
287 outbuf_flags |= postproc->deinterlace ? (
289 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD :
290 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD) :
291 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
292 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
294 GST_BUFFER_TIMESTAMP(outbuf) = timestamp;
295 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
296 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
297 ret = gst_pad_push(postproc->srcpad, outbuf);
298 if (ret != GST_FLOW_OK)
299 goto error_push_buffer;
302 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
304 goto error_create_buffer;
306 meta = gst_buffer_get_vaapi_video_meta(outbuf);
307 outbuf_flags = flags;
308 outbuf_flags |= postproc->deinterlace ? (
310 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD :
311 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD) :
312 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
313 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
315 GST_BUFFER_TIMESTAMP(outbuf) = timestamp + postproc->field_duration;
316 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
317 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
318 ret = gst_pad_push(postproc->srcpad, outbuf);
319 if (ret != GST_FLOW_OK)
320 goto error_push_buffer;
322 gst_buffer_unref(buf);
326 error_invalid_buffer:
328 GST_ERROR("failed to receive a valid video buffer");
329 gst_buffer_unref(buf);
330 return GST_FLOW_UNEXPECTED;
334 GST_ERROR("failed to create output buffer");
335 gst_buffer_unref(buf);
336 return GST_FLOW_UNEXPECTED;
340 if (ret != GST_FLOW_WRONG_STATE)
341 GST_ERROR("failed to push output buffer to video sink");
342 gst_buffer_unref(buf);
343 return GST_FLOW_UNEXPECTED;
348 gst_vaapipostproc_update_sink_caps(GstVaapiPostproc *postproc, GstCaps *caps)
353 if (!gst_video_parse_caps_framerate(caps, &fps_n, &fps_d))
355 postproc->fps_n = fps_n;
356 postproc->fps_d = fps_d;
358 switch (postproc->deinterlace_mode) {
359 case GST_VAAPI_DEINTERLACE_MODE_AUTO:
360 if (!gst_video_format_parse_caps_interlaced(caps, &interlaced))
362 postproc->deinterlace = interlaced;
364 case GST_VAAPI_DEINTERLACE_MODE_INTERLACED:
365 postproc->deinterlace = TRUE;
367 case GST_VAAPI_DEINTERLACE_MODE_DISABLED:
368 postproc->deinterlace = FALSE;
372 postproc->field_duration = gst_util_uint64_scale(
375 (1 + postproc->deinterlace) * postproc->fps_n
378 gst_caps_replace(&postproc->sinkpad_caps, caps);
383 gst_vaapipostproc_update_src_caps(GstVaapiPostproc *postproc, GstCaps *caps)
386 GstStructure *structure;
387 const GValue *v_width, *v_height, *v_par;
390 if (postproc->srcpad_caps)
391 src_caps = gst_caps_make_writable(postproc->srcpad_caps);
393 src_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS_NAME);
396 postproc->srcpad_caps = src_caps;
398 structure = gst_caps_get_structure(caps, 0);
399 v_width = gst_structure_get_value(structure, "width");
400 v_height = gst_structure_get_value(structure, "height");
401 v_par = gst_structure_get_value(structure, "pixel-aspect-ratio");
403 structure = gst_caps_get_structure(src_caps, 0);
404 if (v_width && v_height) {
405 gst_structure_set_value(structure, "width", v_width);
406 gst_structure_set_value(structure, "height", v_height);
409 gst_structure_set_value(structure, "pixel-aspect-ratio", v_par);
411 gst_structure_set(structure, "type", G_TYPE_STRING, "vaapi", NULL);
412 gst_structure_set(structure, "opengl", G_TYPE_BOOLEAN, USE_GLX, NULL);
414 if (!postproc->deinterlace)
415 gst_structure_remove_field(structure, "interlaced");
417 /* Set double framerate in interlaced mode */
418 if (!gst_util_fraction_multiply(postproc->fps_n, postproc->fps_d,
425 "interlaced", G_TYPE_BOOLEAN, FALSE,
426 "framerate", GST_TYPE_FRACTION, fps_n, fps_d,
430 return gst_pad_set_caps(postproc->srcpad, src_caps);
434 gst_vaapipostproc_ensure_allowed_caps(GstVaapiPostproc *postproc)
436 if (postproc->allowed_caps)
439 postproc->allowed_caps =
440 gst_caps_from_string(gst_vaapipostproc_sink_caps_str);
441 if (!postproc->allowed_caps)
444 /* XXX: append VA/VPP filters */
449 gst_vaapipostproc_get_caps(GstPad *pad)
451 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
454 if (gst_vaapipostproc_ensure_allowed_caps(postproc))
455 out_caps = gst_caps_ref(postproc->allowed_caps);
457 out_caps = gst_caps_new_empty();
459 gst_object_unref(postproc);
464 gst_vaapipostproc_set_caps(GstPad *pad, GstCaps *caps)
466 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
467 gboolean success = FALSE;
469 g_return_val_if_fail(pad == postproc->sinkpad, FALSE);
472 if (!gst_vaapipostproc_update_sink_caps(postproc, caps))
474 if (!gst_vaapipostproc_update_src_caps(postproc, caps))
476 if (!gst_vaapipostproc_reset(postproc, postproc->sinkpad_caps))
480 gst_object_unref(postproc);
485 gst_vaapipostproc_chain(GstPad *pad, GstBuffer *buf)
487 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
490 ret = gst_vaapipostproc_process(postproc, buf);
491 gst_object_unref(postproc);
496 gst_vaapipostproc_sink_event(GstPad *pad, GstEvent *event)
498 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
501 GST_DEBUG("handle sink event '%s'", GST_EVENT_TYPE_NAME(event));
503 /* Propagate event downstream */
504 success = gst_pad_push_event(postproc->srcpad, event);
505 gst_object_unref(postproc);
510 gst_vaapipostproc_src_event(GstPad *pad, GstEvent *event)
512 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
515 GST_DEBUG("handle src event '%s'", GST_EVENT_TYPE_NAME(event));
517 /* Propagate event upstream */
518 success = gst_pad_push_event(postproc->sinkpad, event);
519 gst_object_unref(postproc);
524 gst_vaapipostproc_query(GstPad *pad, GstQuery *query)
526 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
529 GST_DEBUG("sharing display %p", postproc->display);
531 if (gst_vaapi_reply_to_query(query, postproc->display))
534 success = gst_pad_query_default(pad, query);
536 gst_object_unref(postproc);
541 gst_vaapipostproc_finalize(GObject *object)
543 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
545 gst_vaapipostproc_destroy(postproc);
547 gst_caps_replace(&postproc->sinkpad_caps, NULL);
548 gst_caps_replace(&postproc->srcpad_caps, NULL);
549 gst_caps_replace(&postproc->allowed_caps, NULL);
551 G_OBJECT_CLASS(gst_vaapipostproc_parent_class)->finalize(object);
555 gst_vaapipostproc_set_property(
562 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
565 case PROP_DEINTERLACE_MODE:
566 postproc->deinterlace_mode = g_value_get_enum(value);
568 case PROP_DEINTERLACE_METHOD:
569 postproc->deinterlace_method = g_value_get_enum(value);
572 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
578 gst_vaapipostproc_get_property(
585 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
588 case PROP_DEINTERLACE_MODE:
589 g_value_set_enum(value, postproc->deinterlace_mode);
591 case PROP_DEINTERLACE_METHOD:
592 g_value_set_enum(value, postproc->deinterlace_method);
595 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
600 static GstStateChangeReturn
601 gst_vaapipostproc_change_state(GstElement *element, GstStateChange transition)
603 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(element);
604 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
606 switch (transition) {
607 case GST_STATE_CHANGE_NULL_TO_READY:
608 if (!gst_vaapipostproc_start(postproc))
609 return GST_STATE_CHANGE_FAILURE;
611 case GST_STATE_CHANGE_READY_TO_PAUSED:
613 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
619 ret = GST_ELEMENT_CLASS(gst_vaapipostproc_parent_class)->change_state(element, transition);
620 if (ret != GST_STATE_CHANGE_SUCCESS)
623 switch (transition) {
624 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
626 case GST_STATE_CHANGE_PAUSED_TO_READY:
628 case GST_STATE_CHANGE_READY_TO_NULL:
629 if (!gst_vaapipostproc_stop(postproc))
630 return GST_STATE_CHANGE_FAILURE;
635 return GST_STATE_CHANGE_SUCCESS;
639 gst_vaapipostproc_class_init(GstVaapiPostprocClass *klass)
641 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
642 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
643 GstPadTemplate *pad_template;
645 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapipostproc,
646 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
648 object_class->finalize = gst_vaapipostproc_finalize;
649 object_class->set_property = gst_vaapipostproc_set_property;
650 object_class->get_property = gst_vaapipostproc_get_property;
652 element_class->change_state = gst_vaapipostproc_change_state;
654 gst_element_class_set_details_simple(
656 gst_vaapipostproc_details.longname,
657 gst_vaapipostproc_details.klass,
658 gst_vaapipostproc_details.description,
659 gst_vaapipostproc_details.author
663 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_sink_factory);
664 gst_element_class_add_pad_template(element_class, pad_template);
665 gst_object_unref(pad_template);
668 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_src_factory);
669 gst_element_class_add_pad_template(element_class, pad_template);
670 gst_object_unref(pad_template);
673 * GstVaapiPostproc:deinterlace-mode:
675 * This selects whether the deinterlacing should always be applied or if
676 * they should only be applied on content that has the "interlaced" flag
679 g_object_class_install_property
681 PROP_DEINTERLACE_MODE,
682 g_param_spec_enum("deinterlace",
684 "Deinterlace mode to use",
685 GST_VAAPI_TYPE_DEINTERLACE_MODE,
686 DEFAULT_DEINTERLACE_MODE,
687 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
690 * GstVaapiPostproc:deinterlace-method:
692 * This selects the deinterlacing method to apply.
694 g_object_class_install_property
696 PROP_DEINTERLACE_METHOD,
697 g_param_spec_enum("deinterlace-method",
698 "Deinterlace method",
699 "Deinterlace method to use",
700 GST_VAAPI_TYPE_DEINTERLACE_METHOD,
701 DEFAULT_DEINTERLACE_METHOD,
702 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
706 gst_vaapipostproc_init(GstVaapiPostproc *postproc)
708 GstVaapiPostprocClass *klass = GST_VAAPIPOSTPROC_GET_CLASS(postproc);
709 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
711 postproc->allowed_caps = NULL;
712 postproc->postproc_caps = NULL;
713 postproc->display = NULL;
714 postproc->surface_width = 0;
715 postproc->surface_height = 0;
716 postproc->deinterlace = FALSE;
717 postproc->deinterlace_mode = DEFAULT_DEINTERLACE_MODE;
718 postproc->deinterlace_method = DEFAULT_DEINTERLACE_METHOD;
719 postproc->field_duration = GST_CLOCK_TIME_NONE;
723 /* Pad through which data comes in to the element */
724 postproc->sinkpad = gst_pad_new_from_template(
725 gst_element_class_get_pad_template(element_class, "sink"),
728 postproc->sinkpad_caps = NULL;
730 gst_pad_set_getcaps_function(postproc->sinkpad, gst_vaapipostproc_get_caps);
731 gst_pad_set_setcaps_function(postproc->sinkpad, gst_vaapipostproc_set_caps);
732 gst_pad_set_chain_function(postproc->sinkpad, gst_vaapipostproc_chain);
733 gst_pad_set_event_function(postproc->sinkpad, gst_vaapipostproc_sink_event);
734 gst_pad_set_query_function(postproc->sinkpad, gst_vaapipostproc_query);
735 gst_element_add_pad(GST_ELEMENT(postproc), postproc->sinkpad);
737 /* Pad through which data goes out of the element */
738 postproc->srcpad = gst_pad_new_from_template(
739 gst_element_class_get_pad_template(element_class, "src"),
742 postproc->srcpad_caps = NULL;
744 gst_pad_set_event_function(postproc->srcpad, gst_vaapipostproc_src_event);
745 gst_pad_set_query_function(postproc->srcpad, gst_vaapipostproc_query);
746 gst_element_add_pad(GST_ELEMENT(postproc), postproc->srcpad);