2 * gstvaapipostproc.c - VA-API video postprocessing
4 * Copyright (C) 2012-2013 Intel Corporation
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either version 2.1
9 * of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 * Boston, MA 02110-1301 USA
23 * SECTION:gstvaapipostproc
24 * @short_description: A video postprocessing filter
26 * vaapipostproc consists in various postprocessing algorithms to be
27 * applied to VA surfaces. So far, only basic bob deinterlacing is
31 #include "gst/vaapi/sysdeps.h"
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
35 #include "gstvaapipostproc.h"
36 #include "gstvaapipluginutil.h"
37 #include "gstvaapivideobuffer.h"
39 #define GST_PLUGIN_NAME "vaapipostproc"
40 #define GST_PLUGIN_DESC "A video postprocessing filter"
42 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapipostproc);
43 #define GST_CAT_DEFAULT gst_debug_vaapipostproc
45 /* Default templates */
46 static const char gst_vaapipostproc_sink_caps_str[] =
47 GST_VAAPI_SURFACE_CAPS ", "
48 "interlaced = (boolean) { true, false }";
50 static const char gst_vaapipostproc_src_caps_str[] =
51 GST_VAAPI_SURFACE_CAPS ", "
52 "interlaced = (boolean) false";
54 static GstStaticPadTemplate gst_vaapipostproc_sink_factory =
55 GST_STATIC_PAD_TEMPLATE(
59 GST_STATIC_CAPS(gst_vaapipostproc_sink_caps_str));
61 static GstStaticPadTemplate gst_vaapipostproc_src_factory =
62 GST_STATIC_PAD_TEMPLATE(
66 GST_STATIC_CAPS(gst_vaapipostproc_src_caps_str));
68 /* GstImplementsInterface interface */
70 gst_vaapipostproc_implements_interface_supported(
71 GstImplementsInterface *iface,
75 return (type == GST_TYPE_VIDEO_CONTEXT);
79 gst_vaapipostproc_implements_iface_init(GstImplementsInterfaceClass *iface)
81 iface->supported = gst_vaapipostproc_implements_interface_supported;
84 /* GstVideoContext interface */
86 gst_vaapipostproc_set_video_context(
87 GstVideoContext *context,
92 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(context);
94 gst_vaapi_set_display(type, value, &postproc->display);
98 gst_video_context_interface_init(GstVideoContextInterface *iface)
100 iface->set_context = gst_vaapipostproc_set_video_context;
103 #define GstVideoContextClass GstVideoContextInterface
104 G_DEFINE_TYPE_WITH_CODE(
108 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
109 gst_vaapipostproc_implements_iface_init);
110 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
111 gst_video_context_interface_init))
116 PROP_DEINTERLACE_MODE,
117 PROP_DEINTERLACE_METHOD,
120 #define DEFAULT_DEINTERLACE_MODE GST_VAAPI_DEINTERLACE_MODE_AUTO
121 #define DEFAULT_DEINTERLACE_METHOD GST_VAAPI_DEINTERLACE_METHOD_BOB
123 #define GST_VAAPI_TYPE_DEINTERLACE_MODE \
124 gst_vaapi_deinterlace_mode_get_type()
127 gst_vaapi_deinterlace_mode_get_type(void)
129 static GType deinterlace_mode_type = 0;
131 static const GEnumValue mode_types[] = {
132 { GST_VAAPI_DEINTERLACE_MODE_AUTO,
133 "Auto detection", "auto" },
134 { GST_VAAPI_DEINTERLACE_MODE_INTERLACED,
135 "Force deinterlacing", "interlaced" },
136 { GST_VAAPI_DEINTERLACE_MODE_DISABLED,
137 "Never deinterlace", "disabled" },
141 if (!deinterlace_mode_type) {
142 deinterlace_mode_type =
143 g_enum_register_static("GstVaapiDeinterlaceMode", mode_types);
145 return deinterlace_mode_type;
148 #define GST_VAAPI_TYPE_DEINTERLACE_METHOD \
149 gst_vaapi_deinterlace_method_get_type()
152 gst_vaapi_deinterlace_method_get_type(void)
154 static GType deinterlace_method_type = 0;
156 static const GEnumValue method_types[] = {
157 { GST_VAAPI_DEINTERLACE_METHOD_BOB,
158 "Bob deinterlacing", "bob" },
161 { GST_VAAPI_DEINTERLACE_METHOD_WEAVE,
162 "Weave deinterlacing", "weave" },
163 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_ADAPTIVE,
164 "Motion adaptive deinterlacing", "motion-adaptive" },
165 { GST_VAAPI_DEINTERLACE_METHOD_MOTION_COMPENSATED,
166 "Motion compensated deinterlacing", "motion-compensated" },
171 if (!deinterlace_method_type) {
172 deinterlace_method_type =
173 g_enum_register_static("GstVaapiDeinterlaceMethod", method_types);
175 return deinterlace_method_type;
178 static inline GstVaapiPostproc *
179 get_vaapipostproc_from_pad(GstPad *pad)
181 return GST_VAAPIPOSTPROC(gst_pad_get_parent_element(pad));
184 static inline gboolean
185 gst_vaapipostproc_ensure_display(GstVaapiPostproc *postproc)
187 return gst_vaapi_ensure_display(postproc, GST_VAAPI_DISPLAY_TYPE_ANY,
192 gst_vaapipostproc_create(GstVaapiPostproc *postproc, GstCaps *caps)
194 if (!gst_vaapipostproc_ensure_display(postproc))
197 gst_caps_replace(&postproc->postproc_caps, caps);
202 gst_vaapipostproc_destroy(GstVaapiPostproc *postproc)
204 gst_caps_replace(&postproc->postproc_caps, NULL);
206 g_clear_object(&postproc->display);
210 gst_vaapipostproc_reset(GstVaapiPostproc *postproc, GstCaps *caps)
212 if (postproc->postproc_caps &&
213 gst_caps_is_always_compatible(caps, postproc->postproc_caps))
216 gst_vaapipostproc_destroy(postproc);
217 return gst_vaapipostproc_create(postproc, caps);
221 gst_vaapipostproc_start(GstVaapiPostproc *postproc)
223 if (!gst_vaapipostproc_ensure_display(postproc))
229 gst_vaapipostproc_stop(GstVaapiPostproc *postproc)
231 if (postproc->display) {
232 g_object_unref(postproc->display);
233 postproc->display = NULL;
239 gst_vaapipostproc_process(GstVaapiPostproc *postproc, GstBuffer *buf)
241 GstVaapiVideoMeta *meta;
242 GstVaapiSurfaceProxy *proxy;
243 GstClockTime timestamp;
245 GstBuffer *outbuf = NULL;
246 guint outbuf_flags, flags;
249 meta = gst_buffer_get_vaapi_video_meta(buf);
251 goto error_invalid_buffer;
253 flags = gst_vaapi_video_meta_get_render_flags(meta);
255 /* Deinterlacing disabled, push frame */
256 if (!postproc->deinterlace) {
257 gst_vaapi_video_meta_set_render_flags(meta, flags);
258 ret = gst_pad_push(postproc->srcpad, buf);
259 if (ret != GST_FLOW_OK)
260 goto error_push_buffer;
264 timestamp = GST_BUFFER_TIMESTAMP(buf);
265 proxy = gst_vaapi_video_meta_get_surface_proxy(meta);
266 tff = GST_BUFFER_FLAG_IS_SET(buf, GST_VIDEO_BUFFER_TFF);
268 flags &= ~(GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD|
269 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD);
272 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
274 goto error_create_buffer;
276 meta = gst_buffer_get_vaapi_video_meta(outbuf);
277 outbuf_flags = flags;
278 outbuf_flags |= postproc->deinterlace ? (
280 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD :
281 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD) :
282 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
283 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
285 GST_BUFFER_TIMESTAMP(outbuf) = timestamp;
286 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
287 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
288 ret = gst_pad_push(postproc->srcpad, outbuf);
289 if (ret != GST_FLOW_OK)
290 goto error_push_buffer;
293 outbuf = gst_vaapi_video_buffer_new_with_surface_proxy(proxy);
295 goto error_create_buffer;
297 meta = gst_buffer_get_vaapi_video_meta(outbuf);
298 outbuf_flags = flags;
299 outbuf_flags |= postproc->deinterlace ? (
301 GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD :
302 GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD) :
303 GST_VAAPI_PICTURE_STRUCTURE_FRAME;
304 gst_vaapi_video_meta_set_render_flags(meta, outbuf_flags);
306 GST_BUFFER_TIMESTAMP(outbuf) = timestamp + postproc->field_duration;
307 GST_BUFFER_DURATION(outbuf) = postproc->field_duration;
308 gst_buffer_set_caps(outbuf, postproc->srcpad_caps);
309 ret = gst_pad_push(postproc->srcpad, outbuf);
310 if (ret != GST_FLOW_OK)
311 goto error_push_buffer;
313 gst_buffer_unref(buf);
317 error_invalid_buffer:
319 GST_ERROR("failed to receive a valid video buffer");
320 gst_buffer_unref(buf);
321 return GST_FLOW_UNEXPECTED;
325 GST_ERROR("failed to create output buffer");
326 gst_buffer_unref(buf);
327 return GST_FLOW_UNEXPECTED;
331 if (ret != GST_FLOW_WRONG_STATE)
332 GST_ERROR("failed to push output buffer to video sink");
333 gst_buffer_unref(buf);
334 return GST_FLOW_UNEXPECTED;
339 gst_vaapipostproc_update_sink_caps(GstVaapiPostproc *postproc, GstCaps *caps)
344 if (!gst_video_parse_caps_framerate(caps, &fps_n, &fps_d))
346 postproc->fps_n = fps_n;
347 postproc->fps_d = fps_d;
349 switch (postproc->deinterlace_mode) {
350 case GST_VAAPI_DEINTERLACE_MODE_AUTO:
351 if (!gst_video_format_parse_caps_interlaced(caps, &interlaced))
353 postproc->deinterlace = interlaced;
355 case GST_VAAPI_DEINTERLACE_MODE_INTERLACED:
356 postproc->deinterlace = TRUE;
358 case GST_VAAPI_DEINTERLACE_MODE_DISABLED:
359 postproc->deinterlace = FALSE;
363 postproc->field_duration = gst_util_uint64_scale(
366 (1 + postproc->deinterlace) * postproc->fps_n
369 gst_caps_replace(&postproc->sinkpad_caps, caps);
374 gst_vaapipostproc_update_src_caps(GstVaapiPostproc *postproc, GstCaps *caps)
377 GstStructure *structure;
378 const GValue *v_width, *v_height, *v_par;
381 if (postproc->srcpad_caps)
382 src_caps = gst_caps_make_writable(postproc->srcpad_caps);
384 src_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS_NAME);
387 postproc->srcpad_caps = src_caps;
389 structure = gst_caps_get_structure(caps, 0);
390 v_width = gst_structure_get_value(structure, "width");
391 v_height = gst_structure_get_value(structure, "height");
392 v_par = gst_structure_get_value(structure, "pixel-aspect-ratio");
394 structure = gst_caps_get_structure(src_caps, 0);
395 if (v_width && v_height) {
396 gst_structure_set_value(structure, "width", v_width);
397 gst_structure_set_value(structure, "height", v_height);
400 gst_structure_set_value(structure, "pixel-aspect-ratio", v_par);
402 gst_structure_set(structure, "type", G_TYPE_STRING, "vaapi", NULL);
403 gst_structure_set(structure, "opengl", G_TYPE_BOOLEAN, USE_GLX, NULL);
405 if (!postproc->deinterlace)
406 gst_structure_remove_field(structure, "interlaced");
408 /* Set double framerate in interlaced mode */
409 if (!gst_util_fraction_multiply(postproc->fps_n, postproc->fps_d,
416 "interlaced", G_TYPE_BOOLEAN, FALSE,
417 "framerate", GST_TYPE_FRACTION, fps_n, fps_d,
421 return gst_pad_set_caps(postproc->srcpad, src_caps);
425 gst_vaapipostproc_ensure_allowed_caps(GstVaapiPostproc *postproc)
427 if (postproc->allowed_caps)
430 postproc->allowed_caps =
431 gst_caps_from_string(gst_vaapipostproc_sink_caps_str);
432 if (!postproc->allowed_caps)
435 /* XXX: append VA/VPP filters */
440 gst_vaapipostproc_get_caps(GstPad *pad)
442 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
445 if (gst_vaapipostproc_ensure_allowed_caps(postproc))
446 out_caps = gst_caps_ref(postproc->allowed_caps);
448 out_caps = gst_caps_new_empty();
450 gst_object_unref(postproc);
455 gst_vaapipostproc_set_caps(GstPad *pad, GstCaps *caps)
457 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
458 gboolean success = FALSE;
460 g_return_val_if_fail(pad == postproc->sinkpad, FALSE);
463 if (!gst_vaapipostproc_update_sink_caps(postproc, caps))
465 if (!gst_vaapipostproc_update_src_caps(postproc, caps))
467 if (!gst_vaapipostproc_reset(postproc, postproc->sinkpad_caps))
471 gst_object_unref(postproc);
476 gst_vaapipostproc_chain(GstPad *pad, GstBuffer *buf)
478 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
481 ret = gst_vaapipostproc_process(postproc, buf);
482 gst_object_unref(postproc);
487 gst_vaapipostproc_sink_event(GstPad *pad, GstEvent *event)
489 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
492 GST_DEBUG("handle sink event '%s'", GST_EVENT_TYPE_NAME(event));
494 /* Propagate event downstream */
495 success = gst_pad_push_event(postproc->srcpad, event);
496 gst_object_unref(postproc);
501 gst_vaapipostproc_src_event(GstPad *pad, GstEvent *event)
503 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
506 GST_DEBUG("handle src event '%s'", GST_EVENT_TYPE_NAME(event));
508 /* Propagate event upstream */
509 success = gst_pad_push_event(postproc->sinkpad, event);
510 gst_object_unref(postproc);
515 gst_vaapipostproc_query(GstPad *pad, GstQuery *query)
517 GstVaapiPostproc * const postproc = get_vaapipostproc_from_pad(pad);
520 GST_DEBUG("sharing display %p", postproc->display);
522 if (gst_vaapi_reply_to_query(query, postproc->display))
525 success = gst_pad_query_default(pad, query);
527 gst_object_unref(postproc);
532 gst_vaapipostproc_finalize(GObject *object)
534 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
536 gst_vaapipostproc_destroy(postproc);
538 gst_caps_replace(&postproc->sinkpad_caps, NULL);
539 gst_caps_replace(&postproc->srcpad_caps, NULL);
540 gst_caps_replace(&postproc->allowed_caps, NULL);
542 G_OBJECT_CLASS(gst_vaapipostproc_parent_class)->finalize(object);
546 gst_vaapipostproc_set_property(
553 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
556 case PROP_DEINTERLACE_MODE:
557 postproc->deinterlace_mode = g_value_get_enum(value);
559 case PROP_DEINTERLACE_METHOD:
560 postproc->deinterlace_method = g_value_get_enum(value);
563 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
569 gst_vaapipostproc_get_property(
576 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(object);
579 case PROP_DEINTERLACE_MODE:
580 g_value_set_enum(value, postproc->deinterlace_mode);
582 case PROP_DEINTERLACE_METHOD:
583 g_value_set_enum(value, postproc->deinterlace_method);
586 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
591 static GstStateChangeReturn
592 gst_vaapipostproc_change_state(GstElement *element, GstStateChange transition)
594 GstVaapiPostproc * const postproc = GST_VAAPIPOSTPROC(element);
595 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
597 switch (transition) {
598 case GST_STATE_CHANGE_NULL_TO_READY:
599 if (!gst_vaapipostproc_start(postproc))
600 return GST_STATE_CHANGE_FAILURE;
602 case GST_STATE_CHANGE_READY_TO_PAUSED:
604 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
610 ret = GST_ELEMENT_CLASS(gst_vaapipostproc_parent_class)->change_state(element, transition);
611 if (ret != GST_STATE_CHANGE_SUCCESS)
614 switch (transition) {
615 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
617 case GST_STATE_CHANGE_PAUSED_TO_READY:
619 case GST_STATE_CHANGE_READY_TO_NULL:
620 if (!gst_vaapipostproc_stop(postproc))
621 return GST_STATE_CHANGE_FAILURE;
626 return GST_STATE_CHANGE_SUCCESS;
630 gst_vaapipostproc_class_init(GstVaapiPostprocClass *klass)
632 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
633 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
634 GstPadTemplate *pad_template;
636 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapipostproc,
637 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
639 object_class->finalize = gst_vaapipostproc_finalize;
640 object_class->set_property = gst_vaapipostproc_set_property;
641 object_class->get_property = gst_vaapipostproc_get_property;
643 element_class->change_state = gst_vaapipostproc_change_state;
645 gst_element_class_set_static_metadata(element_class,
646 "VA-API video postprocessing",
647 "Filter/Converter/Video",
649 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
652 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_sink_factory);
653 gst_element_class_add_pad_template(element_class, pad_template);
656 pad_template = gst_static_pad_template_get(&gst_vaapipostproc_src_factory);
657 gst_element_class_add_pad_template(element_class, pad_template);
660 * GstVaapiPostproc:deinterlace-mode:
662 * This selects whether the deinterlacing should always be applied or if
663 * they should only be applied on content that has the "interlaced" flag
666 g_object_class_install_property
668 PROP_DEINTERLACE_MODE,
669 g_param_spec_enum("deinterlace",
671 "Deinterlace mode to use",
672 GST_VAAPI_TYPE_DEINTERLACE_MODE,
673 DEFAULT_DEINTERLACE_MODE,
674 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
677 * GstVaapiPostproc:deinterlace-method:
679 * This selects the deinterlacing method to apply.
681 g_object_class_install_property
683 PROP_DEINTERLACE_METHOD,
684 g_param_spec_enum("deinterlace-method",
685 "Deinterlace method",
686 "Deinterlace method to use",
687 GST_VAAPI_TYPE_DEINTERLACE_METHOD,
688 DEFAULT_DEINTERLACE_METHOD,
689 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
693 gst_vaapipostproc_init(GstVaapiPostproc *postproc)
695 GstVaapiPostprocClass *klass = GST_VAAPIPOSTPROC_GET_CLASS(postproc);
696 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
698 postproc->allowed_caps = NULL;
699 postproc->postproc_caps = NULL;
700 postproc->display = NULL;
701 postproc->surface_width = 0;
702 postproc->surface_height = 0;
703 postproc->deinterlace = FALSE;
704 postproc->deinterlace_mode = DEFAULT_DEINTERLACE_MODE;
705 postproc->deinterlace_method = DEFAULT_DEINTERLACE_METHOD;
706 postproc->field_duration = GST_CLOCK_TIME_NONE;
710 /* Pad through which data comes in to the element */
711 postproc->sinkpad = gst_pad_new_from_template(
712 gst_element_class_get_pad_template(element_class, "sink"),
715 postproc->sinkpad_caps = NULL;
717 gst_pad_set_getcaps_function(postproc->sinkpad, gst_vaapipostproc_get_caps);
718 gst_pad_set_setcaps_function(postproc->sinkpad, gst_vaapipostproc_set_caps);
719 gst_pad_set_chain_function(postproc->sinkpad, gst_vaapipostproc_chain);
720 gst_pad_set_event_function(postproc->sinkpad, gst_vaapipostproc_sink_event);
721 gst_pad_set_query_function(postproc->sinkpad, gst_vaapipostproc_query);
722 gst_element_add_pad(GST_ELEMENT(postproc), postproc->sinkpad);
724 /* Pad through which data goes out of the element */
725 postproc->srcpad = gst_pad_new_from_template(
726 gst_element_class_get_pad_template(element_class, "src"),
729 postproc->srcpad_caps = NULL;
731 gst_pad_set_event_function(postproc->srcpad, gst_vaapipostproc_src_event);
732 gst_pad_set_query_function(postproc->srcpad, gst_vaapipostproc_query);
733 gst_element_add_pad(GST_ELEMENT(postproc), postproc->srcpad);