3 * Copyright (C) 2016 Freescale Semiconductor, Inc. All rights reserved.
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
31 #include <QtGui/QGuiApplication>
33 #define GST_CAT_DEFAULT gst_debug_qt_gl_src
34 GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
36 #define DEFAULT_IS_LIVE TRUE
38 static void gst_qt_src_set_property (GObject * object, guint prop_id,
39 const GValue * value, GParamSpec * pspec);
40 static void gst_qt_src_get_property (GObject * object, guint prop_id,
41 GValue * value, GParamSpec * pspec);
43 static void gst_qt_src_finalize (GObject * object);
45 static gboolean gst_qt_src_setcaps (GstBaseSrc * bsrc, GstCaps * caps);
46 static GstCaps *gst_qt_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter);
47 static gboolean gst_qt_src_query (GstBaseSrc * bsrc, GstQuery * query);
49 static gboolean gst_qt_src_decide_allocation (GstBaseSrc * bsrc,
51 static GstFlowReturn gst_qt_src_fill (GstPushSrc * psrc, GstBuffer * buffer);
52 static GstStateChangeReturn gst_qt_src_change_state (GstElement * element,
53 GstStateChange transition);
54 static gboolean gst_qt_src_start (GstBaseSrc * basesrc);
55 static gboolean gst_qt_src_stop (GstBaseSrc * basesrc);
57 static GstStaticPadTemplate gst_qt_src_template =
58 GST_STATIC_PAD_TEMPLATE ("src",
61 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
62 "format = (string) RGBA, "
63 "width = " GST_VIDEO_SIZE_RANGE ", "
64 "height = " GST_VIDEO_SIZE_RANGE ", "
65 "framerate = " GST_VIDEO_FPS_RANGE ", "
66 "texture-target = (string) 2D"));
75 #define gst_qt_src_parent_class parent_class
76 G_DEFINE_TYPE_WITH_CODE (GstQtSrc, gst_qt_src,
77 GST_TYPE_PUSH_SRC, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
78 "qtsrc", 0, "Qt Video Src"));
80 static const gfloat vertical_flip_matrix[] = {
81 1.0f, 0.0f, 0.0f, 0.0f,
82 0.0f, -1.0f, 0.0f, 1.0f,
83 0.0f, 0.0f, 1.0f, 0.0f,
84 0.0f, 0.0f, 0.0f, 1.0f,
88 gst_qt_src_class_init (GstQtSrcClass * klass)
90 GObjectClass *gobject_class = (GObjectClass *) klass;
91 GstElementClass *gstelement_class = (GstElementClass *) klass;
92 GstBaseSrcClass *gstbasesrc_class = (GstBaseSrcClass *) klass;
93 GstPushSrcClass *gstpushsrc_class = (GstPushSrcClass *) klass;
95 gobject_class->set_property = gst_qt_src_set_property;
96 gobject_class->get_property = gst_qt_src_get_property;
97 gobject_class->finalize = gst_qt_src_finalize;
99 gst_element_class_set_metadata (gstelement_class, "Qt Video Source",
100 "Source/Video", "A video src the grab window from a qml view",
101 "Multimedia Team <shmmmw@freescale.com>");
103 g_object_class_install_property (gobject_class, PROP_WINDOW,
104 g_param_spec_pointer ("window", "QQuickWindow",
105 "The QQuickWindow to place in the object heirachy",
106 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
108 g_object_class_install_property (gobject_class, PROP_DEFAULT_FBO,
109 g_param_spec_boolean ("use-default-fbo",
110 "If use default fbo",
111 "When set TRUE, it will not create new fbo for qml render thread",
112 FALSE, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
114 gst_element_class_add_pad_template (gstelement_class,
115 gst_static_pad_template_get (&gst_qt_src_template));
117 gstelement_class->change_state = gst_qt_src_change_state;
118 gstbasesrc_class->set_caps = gst_qt_src_setcaps;
119 gstbasesrc_class->get_caps = gst_qt_src_get_caps;
120 gstbasesrc_class->query = gst_qt_src_query;
121 gstbasesrc_class->start = gst_qt_src_start;
122 gstbasesrc_class->stop = gst_qt_src_stop;
123 gstbasesrc_class->decide_allocation = gst_qt_src_decide_allocation;
125 gstpushsrc_class->fill = gst_qt_src_fill;
129 gst_qt_src_init (GstQtSrc * src)
131 gst_base_src_set_format (GST_BASE_SRC (src), GST_FORMAT_TIME);
132 gst_base_src_set_live (GST_BASE_SRC (src), DEFAULT_IS_LIVE);
133 src->default_fbo = FALSE;
134 src->pending_image_orientation = TRUE;
138 gst_qt_src_set_property (GObject * object, guint prop_id,
139 const GValue * value, GParamSpec * pspec)
141 GstQtSrc *qt_src = GST_QT_SRC (object);
146 static_cast < QQuickWindow * >(g_value_get_pointer (value));
149 delete qt_src->window;
151 qt_src->window = new QtGLWindow (NULL, qt_src->qwindow);
155 case PROP_DEFAULT_FBO:
156 qt_src->default_fbo = g_value_get_boolean (value);
159 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
165 gst_qt_src_get_property (GObject * object, guint prop_id,
166 GValue * value, GParamSpec * pspec)
168 GstQtSrc *qt_src = GST_QT_SRC (object);
172 g_value_set_pointer (value, qt_src->qwindow);
174 case PROP_DEFAULT_FBO:
175 g_value_set_boolean (value, qt_src->default_fbo);
178 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
184 gst_qt_src_finalize (GObject * object)
186 GstQtSrc *qt_src = GST_QT_SRC (object);
188 GST_DEBUG ("qmlglsrc finalize");
190 gst_object_unref (qt_src->context);
191 qt_src->context = NULL;
193 if (qt_src->qt_context)
194 gst_object_unref (qt_src->qt_context);
195 qt_src->qt_context = NULL;
198 gst_object_unref (qt_src->display);
199 qt_src->display = NULL;
202 delete qt_src->window;
204 G_OBJECT_CLASS (parent_class)->finalize (object);
208 gst_qt_src_setcaps (GstBaseSrc * bsrc, GstCaps * caps)
210 GstQtSrc *qt_src = GST_QT_SRC (bsrc);
212 GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps);
214 if (!gst_video_info_from_caps (&qt_src->v_info, caps))
217 if (!qt_window_set_caps (qt_src->window, caps))
224 gst_qt_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter)
226 GstCaps *caps = NULL, *temp = NULL;
227 GstPadTemplate *pad_template;
228 GstBaseSrcClass *bclass = GST_BASE_SRC_GET_CLASS (bsrc);
229 GstQtSrc *qt_src = GST_QT_SRC (bsrc);
233 if (qt_src->window) {
234 qt_src->window->getGeometry (&width, &height);
238 gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "src");
239 if (pad_template != NULL)
240 caps = gst_pad_template_get_caps (pad_template);
242 if (qt_src->window) {
243 temp = gst_caps_copy (caps);
244 guint n_caps = gst_caps_get_size (caps);
246 for (i = 0; i < n_caps; i++) {
247 GstStructure *s = gst_caps_get_structure (temp, i);
248 gst_structure_set (s, "width", G_TYPE_INT, width, NULL);
249 gst_structure_set (s, "height", G_TYPE_INT, height, NULL);
250 /* because the framerate is unknown */
251 gst_structure_set (s, "framerate", GST_TYPE_FRACTION, 0, 1, NULL);
252 gst_structure_set (s, "pixel-aspect-ratio",
253 GST_TYPE_FRACTION, 1, 1, NULL);
256 gst_caps_unref (caps);
261 GstCaps *intersection;
264 gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
265 gst_caps_unref (caps);
273 gst_qt_src_query (GstBaseSrc * bsrc, GstQuery * query)
275 GstQtSrc *qt_src = GST_QT_SRC (bsrc);
276 gboolean res = FALSE;
278 switch (GST_QUERY_TYPE (query)) {
279 case GST_QUERY_CONTEXT:
281 const gchar *context_type;
282 GstContext *context, *old_context;
285 if (!qt_window_is_scenegraph_initialized (qt_src->window))
288 if (!qt_src->display && !qt_src->qt_context) {
289 qt_src->display = qt_window_get_display (qt_src->window);
290 qt_src->qt_context = qt_window_get_qt_context (qt_src->window);
293 ret = gst_gl_handle_context_query ((GstElement *) qt_src, query,
294 &qt_src->display, &qt_src->qt_context);
297 gst_gl_display_filter_gl_api (qt_src->display,
298 gst_gl_context_get_gl_api (qt_src->qt_context));
300 gst_query_parse_context_type (query, &context_type);
302 if (g_strcmp0 (context_type, "gst.gl.app_context") == 0) {
305 gst_query_parse_context (query, &old_context);
308 context = gst_context_copy (old_context);
310 context = gst_context_new ("gst.gl.app_context", FALSE);
312 s = gst_context_writable_structure (context);
313 gst_structure_set (s, "context", GST_TYPE_GL_CONTEXT,
314 qt_src->qt_context, NULL);
315 gst_query_set_context (query, context);
316 gst_context_unref (context);
318 ret = qt_src->qt_context != NULL;
320 GST_LOG_OBJECT (qt_src, "context query of type %s %i", context_type, ret);
328 res = GST_BASE_SRC_CLASS (parent_class)->query (bsrc, query);
336 _find_local_gl_context (GstQtSrc * qt_src)
340 const GstStructure *s;
345 query = gst_query_new_context ("gst.gl.local_context");
347 && gst_gl_run_query (GST_ELEMENT (qt_src), query, GST_PAD_SRC)) {
348 gst_query_parse_context (query, &context);
350 s = gst_context_get_structure (context);
351 gst_structure_get (s, "context", GST_TYPE_GL_CONTEXT, &qt_src->context,
356 GST_DEBUG_OBJECT (qt_src, "found local context %p", qt_src->context);
358 gst_query_unref (query);
367 gst_qt_src_decide_allocation (GstBaseSrc * bsrc, GstQuery * query)
369 GstBufferPool *pool = NULL;
370 GstStructure *config;
372 guint min, max, size, n, i;
373 gboolean update_pool, update_allocator;
374 GstAllocator *allocator;
375 GstAllocationParams params;
376 GstGLVideoAllocationParams *glparams;
378 GstQtSrc *qt_src = GST_QT_SRC (bsrc);
380 if (gst_query_find_allocation_meta (query,
381 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
382 qt_src->downstream_supports_affine_meta = TRUE;
384 qt_src->downstream_supports_affine_meta = FALSE;
387 gst_query_parse_allocation (query, &caps, NULL);
391 gst_video_info_from_caps (&vinfo, caps);
393 n = gst_query_get_n_allocation_pools (query);
396 for (i = 0; i < n; i++) {
397 gst_query_parse_nth_allocation_pool (query, i, &pool, &size, &min, &max);
399 if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) {
401 gst_object_unref (pool);
413 if (!qt_src->context && !_find_local_gl_context (qt_src))
417 if (!qt_src->context || !GST_IS_GL_CONTEXT (qt_src->context))
420 pool = gst_gl_buffer_pool_new (qt_src->context);
421 GST_INFO_OBJECT (qt_src, "No pool, create one ourself %p", pool);
424 config = gst_buffer_pool_get_config (pool);
426 gst_buffer_pool_config_set_params (config, caps, size, min, max);
427 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
428 if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
429 gst_buffer_pool_config_add_option (config,
430 GST_BUFFER_POOL_OPTION_GL_SYNC_META);
432 if (gst_query_get_n_allocation_params (query) > 0) {
433 gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms);
434 gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);
435 GST_INFO_OBJECT (qt_src, "got allocator %p", allocator);
436 update_allocator = TRUE;
439 gst_allocation_params_init (¶ms);
440 update_allocator = FALSE;
444 gst_gl_video_allocation_params_new (qt_src->context, ¶ms, &vinfo, 0,
445 NULL, GST_GL_TEXTURE_TARGET_2D, GST_VIDEO_GL_TEXTURE_TYPE_RGBA);
446 gst_buffer_pool_config_set_gl_allocation_params (config,
447 (GstGLAllocationParams *) glparams);
448 gst_gl_allocation_params_free ((GstGLAllocationParams *) glparams);
450 if (!gst_buffer_pool_set_config (pool, config))
451 GST_WARNING_OBJECT (qt_src, "Failed to set buffer pool config");
453 if (update_allocator)
454 gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms);
456 gst_query_add_allocation_param (query, allocator, ¶ms);
458 gst_object_unref (allocator);
461 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
463 gst_query_add_allocation_pool (query, pool, size, min, max);
464 gst_object_unref (pool);
466 GST_INFO_OBJECT (qt_src, "successfully decide_allocation");
471 gst_qt_src_fill (GstPushSrc * psrc, GstBuffer * buffer)
473 GstQtSrc *qt_src = GST_QT_SRC (psrc);
475 GST_DEBUG_OBJECT (qt_src, "setting buffer %p", buffer);
477 if (!qt_window_set_buffer (qt_src->window, buffer)) {
478 GST_ERROR_OBJECT (qt_src, "failed to fill buffer %p", buffer);
479 return GST_FLOW_ERROR;
482 if (!qt_src->downstream_supports_affine_meta) {
483 if (qt_src->pending_image_orientation) {
484 /* let downstream know the image orientation is vertical filp */
485 GstTagList *image_orientation_tag =
486 gst_tag_list_new (GST_TAG_IMAGE_ORIENTATION, "flip-rotate-180", NULL);
488 gst_pad_push_event (GST_BASE_SRC_PAD (psrc),
489 gst_event_new_tag (image_orientation_tag));
491 qt_src->pending_image_orientation = FALSE;
494 GstVideoAffineTransformationMeta *trans_meta;
495 trans_meta = gst_buffer_add_video_affine_transformation_meta (buffer);
496 gst_video_affine_transformation_meta_apply_matrix (trans_meta,
497 vertical_flip_matrix);
500 GST_DEBUG_OBJECT (qt_src, "buffer fill done %p", buffer);
505 static GstStateChangeReturn
506 gst_qt_src_change_state (GstElement * element, GstStateChange transition)
508 GstQtSrc *qt_src = GST_QT_SRC (element);
509 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
510 QGuiApplication *app;
511 guint64 frames_rendered = 0;
513 GST_DEBUG ("changing state: %s => %s",
514 gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
515 gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
517 switch (transition) {
518 case GST_STATE_CHANGE_NULL_TO_READY:
519 app = static_cast < QGuiApplication * >(QCoreApplication::instance ());
521 GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
522 ("%s", "Failed to connect to Qt"),
523 ("%s", "Could not retrieve QGuiApplication instance"));
524 return GST_STATE_CHANGE_FAILURE;
527 if (!qt_src->window) {
528 GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
529 ("%s", "Required property \'window\' not set"), (NULL));
530 return GST_STATE_CHANGE_FAILURE;
533 if (!qt_window_is_scenegraph_initialized (qt_src->window)) {
534 GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
535 ("%s", "Could not initialize window system"), (NULL));
536 return GST_STATE_CHANGE_FAILURE;
539 qt_window_use_default_fbo (qt_src->window, qt_src->default_fbo);
542 case GST_STATE_CHANGE_READY_TO_PAUSED:
544 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
550 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
551 if (ret == GST_STATE_CHANGE_FAILURE)
554 switch (transition) {
555 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
556 qt_src->run_time = gst_element_get_start_time (GST_ELEMENT (qt_src));
558 case GST_STATE_CHANGE_PAUSED_TO_READY:
560 case GST_STATE_CHANGE_READY_TO_NULL:
561 qt_window_get_total_frames (qt_src->window, &frames_rendered);
562 if (qt_src->run_time > 0) {
563 GST_DEBUG ("qmlglsrc Total refresh frames (%ld), playing for (%"
564 GST_TIME_FORMAT "), fps (%.3f).\n", frames_rendered,
565 GST_TIME_ARGS (qt_src->run_time),
566 (gfloat) GST_SECOND * frames_rendered / qt_src->run_time);
577 gst_qt_src_start (GstBaseSrc * basesrc)
579 GstQtSrc *qt_src = GST_QT_SRC (basesrc);
581 /* already has get OpenGL configuration from qt */
582 if (qt_src->display && qt_src->qt_context)
585 if (!qt_window_is_scenegraph_initialized (qt_src->window))
588 qt_src->display = qt_window_get_display (qt_src->window);
589 qt_src->qt_context = qt_window_get_qt_context (qt_src->window);
591 if (!qt_src->display || !qt_src->qt_context) {
592 GST_ERROR_OBJECT (qt_src,
593 "Could not retrieve window system OpenGL configuration");
597 GST_DEBUG_OBJECT (qt_src, "Got qt display %p and qt gl context %p",
598 qt_src->display, qt_src->qt_context);
603 gst_qt_src_stop (GstBaseSrc * basesrc)