2 * gstvaapipostproc.c - VA-API video postprocessing
4 * Copyright (C) 2012 Intel Corporation
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either version 2.1
9 * of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301 USA
23 * SECTION:gstvaapipostproc
24 * @short_description: A video postprocessing filter
26 * vaapipostproc consists in various postprocessing algorithms to be
27 * applied to VA surfaces. So far, only basic bob deinterlacing is
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
34 #include <gst/vaapi/gstvaapivideosink.h>
35 #include <gst/vaapi/gstvaapivideobuffer.h>
37 #include "gstvaapipluginutil.h"
38 #include "gstvaapipostproc.h"
40 #define GST_PLUGIN_NAME "vaapipostproc"
41 #define GST_PLUGIN_DESC "A video postprocessing filter"
43 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapipostproc);
44 #define GST_CAT_DEFAULT gst_debug_vaapipostproc
46 /* ElementFactory information */
47 static const GstElementDetails gst_vaapipostproc_details =
49 "VA-API video postprocessing",
50 "Filter/Converter/Video",
52 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
54 /* Default templates */
55 static const char gst_vaapipostproc_sink_caps_str[] =
56 GST_VAAPI_SURFACE_CAPS ", "
57 "interlaced = (boolean) { true, false }";
59 static const char gst_vaapipostproc_src_caps_str[] =
60 GST_VAAPI_SURFACE_CAPS ", "
61 "interlaced = (boolean) false";
63 static GstStaticPadTemplate gst_vaapipostproc_sink_factory =
64 GST_STATIC_PAD_TEMPLATE(
68 GST_STATIC_CAPS(gst_vaapipostproc_sink_caps_str));
70 static GstStaticPadTemplate gst_vaapipostproc_src_factory =
71 GST_STATIC_PAD_TEMPLATE(
75 GST_STATIC_CAPS(gst_vaapipostproc_src_caps_str));
78 gst_video_context_interface_init(GstVideoContextInterface *iface);
80 #define GstVideoContextClass GstVideoContextInterface
81 G_DEFINE_TYPE_WITH_CODE(
85 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
86 gst_video_context_interface_init));
91 PROP_DEINTERLACE_MODE,
92 PROP_DEINTERLACE_METHOD,
95 #define DEFAULT_DEINTERLACE_MODE GST_VAAPI_DEINTERLACE_MODE_AUTO
96 #define DEFAULT_DEINTERLACE_METHOD GST_VAAPI_DEINTERLACE_METHOD_BOB
98 #define GST_TYPE_VAAPI_DEINTERLACE_MODES \
99 gst_vaapi_deinterlace_modes_get_type()
102 gst_vaapi_deinterlace_modes_get_type(void)
104 static GType deinterlace_modes_type = 0;
106 static const GEnumValue modes_types[] = {
107 { GST_VAAPI_DEINTERLACE_MODE_AUTO,
108 "Auto detection", "auto" },
109 { GST_VAAPI_DEINTERLACE_MODE_INTERLACED,
110 "Force deinterlacing", "interlaced" },
111 { GST_VAAPI_DEINTERLACE_MODE_DISABLED,
112 "Never deinterlace", "disabled" },
116 if (!deinterlace_modes_type) {
117 deinterlace_modes_type =
118 g_enum_register_static("GstVaapiDeinterlaceModes", modes_types);
120 return deinterlace_modes_type;
123 #define GST_TYPE_VAAPI_DEINTERLACE_METHODS \
124 gst_vaapi_deinterlace_methods_get_type()
127 gst_vaapi_deinterlace_methods_get_type(void)
129 static GType deinterlace_methods_type = 0;
131 static const GEnumValue methods_types[] = {
132 { GST_VAAPI_DEINTERLACE_METHOD_BOB,
133 "Bob deinterlacing", "bob" },
136 { GST_VAAPI_DEINTERLACE_METHOD_WEAVE,
137 "Weave deinterlacing", "weave" },
138 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_ADAPTIVE,
139 "Motion adaptive deinterlacing", "motion-adaptive" },
140 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_COMPENSATED,
141 "Motion compensated deinterlacing", "motion-compensated" },
146 if (!deinterlace_methods_type) {
147 deinterlace_methods_type =
148 g_enum_register_static("GstVaapiDeinterlaceMethods", methods_types);
150 return deinterlace_methods_type;
153 static inline GstVaapiPostproc *
154 get_vaapipostproc_from_pad(GstPad *pad)
156 return GST_VAAPIPOSTPROC(gst_pad_get_parent_element(pad));
159 /* GstVideoContext interface */
162 gst_vaapipostproc_set_video_context(
163 GstVideoContext *context,
168 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(context);
170 gst_vaapi_set_display(type, value, &postproc->display);
174 gst_video_context_supported(GstVaapiPostproc *postproc, GType iface_type)
176 return (iface_type == GST_TYPE_VIDEO_CONTEXT);
180 gst_video_context_interface_init(GstVideoContextInterface *iface)
182 iface->set_context = gst_vaapipostproc_set_video_context;
186 gst_vaapipostproc_create(GstVaapiPostproc *postproc, GstCaps *caps)
188 if (!gst_vaapi_ensure_display(postproc, &postproc->display))
191 gst_caps_replace(&postproc->postproc_caps, caps);
196 gst_vaapipostproc_destroy(GstVaapiPostproc *postproc)
198 gst_caps_replace(&postproc->postproc_caps, NULL);
200 if (postproc->display) {
201 g_object_unref(postproc->display);
202 postproc->display = NULL;
207 gst_vaapipostproc_reset(GstVaapiPostproc *postproc, GstCaps *caps)
209 if (postproc->postproc_caps &&
210 gst_caps_is_always_compatible(caps, postproc->postproc_caps))
213 gst_vaapipostproc_destroy(postproc);
214 return gst_vaapipostproc_create(postproc, caps);
218 gst_vaapipostproc_start(GstVaapiPostproc *postproc)
220 if (!gst_vaapi_ensure_display(postproc, &postproc->display))
226 gst_vaapipostproc_stop(GstVaapiPostproc *postproc)
228 if (postproc->display) {
229 g_object_unref(postproc->display);
230 postproc->display = NULL;
236 gst_vaapipostproc_process(GstVaapiPostproc *postproc, GstBuffer *buf)
238 GstVaapiVideoBuffer *vbuf = GST_VAAPI_VIDEO_BUFFER(buf);
239 GstVaapiSurfaceProxy *proxy;
240 GstClockTime timestamp;
242 GstBuffer *outbuf = NULL;
243 guint outbuf_flags, flags;
244 gboolean interlaced, tff;
246 flags = gst_vaapi_video_buffer_get_render_flags(vbuf);
248 /* Deinterlacing disabled, push frame */
249 if (!postproc->deinterlace) {
250 gst_vaapi_video_buffer_set_render_flags(vbuf, flags);
251 ret = gst_pad_push(postproc->srcpad, buf);
252 if (ret != GST_FLOW_OK)
253 goto error_push_buffer;
257 timestamp = GST_BUFFER_TIMESTAMP(buf);
258 proxy = gst_vaapi_video_buffer_get_surface_proxy(vbuf);
259 interlaced = gst_vaapi_surface_proxy_get_interlaced(proxy);
260 tff = gst_vaapi_surface_proxy_get_tff(proxy);
262 flags &= ~(GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD|
263 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD);
266 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
268 goto error_create_buffer;
270 vbuf = GST_VAAPI_VIDEO_BUFFER(outbuf);
271 outbuf_flags = flags;
272 outbuf_flags |= interlaced ? (
274 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD :
275 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD) :
276 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
277 gst_vaapi_video_buffer_set_render_flags(vbuf, outbuf_flags);
279 GST_BUFFER_TIMESTAMP(outbuf) = timestamp;
280 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
281 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
282 ret = gst_pad_push(postproc->srcpad, outbuf);
283 if (ret != GST_FLOW_OK)
284 goto error_push_buffer;
287 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
289 goto error_create_buffer;
291 vbuf = GST_VAAPI_VIDEO_BUFFER(outbuf);
292 outbuf_flags = flags;
293 outbuf_flags |= interlaced ? (
295 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD :
296 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD) :
297 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
298 gst_vaapi_video_buffer_set_render_flags(vbuf, outbuf_flags);
300 GST_BUFFER_TIMESTAMP(outbuf) = timestamp + postproc->field_duration;
301 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
302 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
303 ret = gst_pad_push(postproc->srcpad, outbuf);
304 if (ret != GST_FLOW_OK)
305 goto error_push_buffer;
307 gst_buffer_unref(buf);
313 GST_ERROR("failed to create output buffer");
314 gst_buffer_unref(buf);
315 return GST_FLOW_UNEXPECTED;
319 if (ret != GST_FLOW_WRONG_STATE)
320 GST_ERROR("failed to push output buffer to video sink");
321 gst_buffer_unref(buf);
322 return GST_FLOW_UNEXPECTED;
327 gst_vaapipostproc_update_sink_caps(GstVaapiPostproc *postproc, GstCaps *caps)
332 if (!gst_video_parse_caps_framerate(caps, &fps_n, &fps_d))
334 postproc->fps_n = fps_n;
335 postproc->fps_d = fps_d;
337 switch (postproc->deinterlace_mode) {
338 case GST_VAAPI_DEINTERLACE_MODE_AUTO:
339 if (!gst_video_format_parse_caps_interlaced(caps, &interlaced))
341 postproc->deinterlace = interlaced;
343 case GST_VAAPI_DEINTERLACE_MODE_INTERLACED:
344 postproc->deinterlace = TRUE;
346 case GST_VAAPI_DEINTERLACE_MODE_DISABLED:
347 postproc->deinterlace = FALSE;
351 postproc->field_duration = gst_util_uint64_scale(
354 (1 + postproc->deinterlace) * postproc->fps_n
357 gst_caps_replace(&postproc->sinkpad_caps, caps);
362 gst_vaapipostproc_update_src_caps(GstVaapiPostproc *postproc, GstCaps *caps)
365 GstStructure *structure;
366 const GValue *v_width, *v_height, *v_par;
369 if (postproc->srcpad_caps)
370 src_caps = gst_caps_make_writable(postproc->srcpad_caps);
372 src_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS_NAME);
375 postproc->srcpad_caps = src_caps;
377 structure = gst_caps_get_structure(caps, 0);
378 v_width = gst_structure_get_value(structure, "width");
379 v_height = gst_structure_get_value(structure, "height");
380 v_par = gst_structure_get_value(structure, "pixel-aspect-ratio");
382 structure = gst_caps_get_structure(src_caps, 0);
383 if (v_width && v_height) {
384 gst_structure_set_value(structure, "width", v_width);
385 gst_structure_set_value(structure, "height", v_height);
388 gst_structure_set_value(structure, "pixel-aspect-ratio", v_par);
390 gst_structure_set(structure, "type", G_TYPE_STRING, "vaapi", NULL);
391 gst_structure_set(structure, "opengl", G_TYPE_BOOLEAN, USE_VAAPI_GLX, NULL);
393 if (!postproc->deinterlace)
394 gst_structure_remove_field(structure, "interlaced");
396 /* Set double framerate in interlaced mode */
397 if (!gst_util_fraction_multiply(postproc->fps_n, postproc->fps_d,
404 "interlaced", G_TYPE_BOOLEAN, FALSE,
405 "framerate", GST_TYPE_FRACTION, fps_n, fps_d,
409 return gst_pad_set_caps(postproc->srcpad, src_caps);
413 gst_vaapipostproc_ensure_allowed_caps(GstVaapiPostproc *postproc)
415 if (postproc->allowed_caps)
418 postproc->allowed_caps =
419 gst_caps_from_string(gst_vaapipostproc_sink_caps_str);
420 if (!postproc->allowed_caps)
423 /* XXX: append VA/VPP filters */
428 gst_vaapipostproc_get_caps(GstPad *pad)
430 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
433 if (gst_vaapipostproc_ensure_allowed_caps(postproc))
434 out_caps = gst_caps_ref(postproc->allowed_caps);
436 out_caps = gst_caps_new_empty();
438 gst_object_unref(postproc);
443 gst_vaapipostproc_set_caps(GstPad *pad, GstCaps *caps)
445 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
446 gboolean success = FALSE;
448 g_return_val_if_fail(pad == postproc->sinkpad, FALSE);
451 if (!gst_vaapipostproc_update_sink_caps(postproc, caps))
453 if (!gst_vaapipostproc_update_src_caps(postproc, caps))
455 if (!gst_vaapipostproc_reset(postproc, postproc->sinkpad_caps))
459 gst_object_unref(postproc);
464 gst_vaapipostproc_chain(GstPad *pad, GstBuffer *buf)
466 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
469 ret = gst_vaapipostproc_process(postproc, buf);
470 gst_object_unref(postproc);
475 gst_vaapipostproc_sink_event(GstPad *pad, GstEvent *event)
477 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
480 GST_DEBUG("handle sink event '%s'", GST_EVENT_TYPE_NAME(event));
482 /* Propagate event downstream */
483 success = gst_pad_push_event(postproc->srcpad, event);
484 gst_object_unref(postproc);
489 gst_vaapipostproc_src_event(GstPad *pad, GstEvent *event)
491 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
494 GST_DEBUG("handle src event '%s'", GST_EVENT_TYPE_NAME(event));
496 /* Propagate event upstream */
497 success = gst_pad_push_event(postproc->sinkpad, event);
498 gst_object_unref(postproc);
503 gst_vaapipostproc_query(GstPad *pad, GstQuery *query)
505 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
508 GST_DEBUG("sharing display %p", postproc->display);
510 if (gst_vaapi_reply_to_query(query, postproc->display))
513 success = gst_pad_query_default(pad, query);
515 gst_object_unref(postproc);
520 gst_vaapipostproc_finalize(GObject *object)
522 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
524 gst_vaapipostproc_destroy(postproc);
526 gst_caps_replace(&postproc->sinkpad_caps, NULL);
527 gst_caps_replace(&postproc->srcpad_caps, NULL);
528 gst_caps_replace(&postproc->allowed_caps, NULL);
530 G_OBJECT_CLASS(gst_vaapipostproc_parent_class)->finalize(object);
534 gst_vaapipostproc_set_property(
541 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
544 case PROP_DEINTERLACE_MODE:
545 postproc->deinterlace_mode = g_value_get_enum(value);
547 case PROP_DEINTERLACE_METHOD:
548 postproc->deinterlace_method = g_value_get_enum(value);
551 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
557 gst_vaapipostproc_get_property(
564 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
567 case PROP_DEINTERLACE_MODE:
568 g_value_set_enum(value, postproc->deinterlace_mode);
570 case PROP_DEINTERLACE_METHOD:
571 g_value_set_enum(value, postproc->deinterlace_method);
574 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
579 static GstStateChangeReturn
580 gst_vaapipostproc_change_state(GstElement *element, GstStateChange transition)
582 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(element);
583 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
585 switch (transition) {
586 case GST_STATE_CHANGE_NULL_TO_READY:
587 if (!gst_vaapipostproc_start(postproc))
588 return GST_STATE_CHANGE_FAILURE;
590 case GST_STATE_CHANGE_READY_TO_PAUSED:
592 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
598 ret = GST_ELEMENT_CLASS(gst_vaapipostproc_parent_class)->change_state(element, transition);
599 if (ret != GST_STATE_CHANGE_SUCCESS)
602 switch (transition) {
603 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
605 case GST_STATE_CHANGE_PAUSED_TO_READY:
607 case GST_STATE_CHANGE_READY_TO_NULL:
608 if (!gst_vaapipostproc_stop(postproc))
609 return GST_STATE_CHANGE_FAILURE;
614 return GST_STATE_CHANGE_SUCCESS;
618 gst_vaapipostproc_class_init(GstVaapiPostprocClass *klass)
620 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
621 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
622 GstPadTemplate *pad_template;
624 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapipostproc,
625 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
627 object_class->finalize = gst_vaapipostproc_finalize;
628 object_class->set_property = gst_vaapipostproc_set_property;
629 object_class->get_property = gst_vaapipostproc_get_property;
631 element_class->change_state = gst_vaapipostproc_change_state;
633 gst_element_class_set_details_simple(
635 gst_vaapipostproc_details.longname,
636 gst_vaapipostproc_details.klass,
637 gst_vaapipostproc_details.description,
638 gst_vaapipostproc_details.author
642 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_sink_factory);
643 gst_element_class_add_pad_template(element_class, pad_template);
644 gst_object_unref(pad_template);
647 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_src_factory);
648 gst_element_class_add_pad_template(element_class, pad_template);
649 gst_object_unref(pad_template);
652 * GstVaapiPostproc:deinterlace-mode:
654 * This selects whether the deinterlacing should always be applied or if
655 * they should only be applied on content that has the "interlaced" flag
658 g_object_class_install_property
660 PROP_DEINTERLACE_MODE,
661 g_param_spec_enum("deinterlace",
663 "Deinterlace mode to use",
664 GST_TYPE_VAAPI_DEINTERLACE_MODES,
665 DEFAULT_DEINTERLACE_MODE,
666 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
669 * GstVaapiPostproc:deinterlace-method:
671 * This selects the deinterlacing method to apply.
673 g_object_class_install_property
675 PROP_DEINTERLACE_METHOD,
676 g_param_spec_enum("deinterlace-method",
677 "Deinterlace method",
678 "Deinterlace method to use",
679 GST_TYPE_VAAPI_DEINTERLACE_METHODS,
680 DEFAULT_DEINTERLACE_METHOD,
681 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
685 gst_vaapipostproc_init(GstVaapiPostproc *postproc)
687 GstVaapiPostprocClass *klass = GST_VAAPIPOSTPROC_GET_CLASS(postproc);
688 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
690 postproc->allowed_caps = NULL;
691 postproc->postproc_caps = NULL;
692 postproc->display = NULL;
693 postproc->surface_width = 0;
694 postproc->surface_height = 0;
695 postproc->deinterlace = FALSE;
696 postproc->deinterlace_mode = DEFAULT_DEINTERLACE_MODE;
697 postproc->deinterlace_method = DEFAULT_DEINTERLACE_METHOD;
698 postproc->field_duration = GST_CLOCK_TIME_NONE;
702 /* Pad through which data comes in to the element */
703 postproc->sinkpad = gst_pad_new_from_template(
704 gst_element_class_get_pad_template(element_class, "sink"),
707 postproc->sinkpad_caps = NULL;
709 gst_pad_set_getcaps_function(postproc->sinkpad, gst_vaapipostproc_get_caps);
710 gst_pad_set_setcaps_function(postproc->sinkpad, gst_vaapipostproc_set_caps);
711 gst_pad_set_chain_function(postproc->sinkpad, gst_vaapipostproc_chain);
712 gst_pad_set_event_function(postproc->sinkpad, gst_vaapipostproc_sink_event);
713 gst_pad_set_query_function(postproc->sinkpad, gst_vaapipostproc_query);
714 gst_element_add_pad(GST_ELEMENT(postproc), postproc->sinkpad);
716 /* Pad through which data goes out of the element */
717 postproc->srcpad = gst_pad_new_from_template(
718 gst_element_class_get_pad_template(element_class, "src"),
721 postproc->srcpad_caps = NULL;
723 gst_pad_set_event_function(postproc->srcpad, gst_vaapipostproc_src_event);
724 gst_pad_set_query_function(postproc->srcpad, gst_vaapipostproc_query);
725 gst_element_add_pad(GST_ELEMENT(postproc), postproc->srcpad);