1 /* Small helper element for format conversion
2 * Copyright (C) 2005 Tim-Philipp Müller <tim centricular net>
3 * Copyright (C) 2010 Brandon Lewis <brandon.lewis@collabora.co.uk>
4 * Copyright (C) 2010 Edward Hervey <edward.hervey@collabora.co.uk>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
25 #include "gst/glib-compat-private.h"
28 caps_are_raw (const GstCaps * caps)
32 len = gst_caps_get_size (caps);
34 for (i = 0; i < len; i++) {
35 GstStructure *st = gst_caps_get_structure (caps, i);
36 if (gst_structure_has_name (st, "video/x-raw-yuv") ||
37 gst_structure_has_name (st, "video/x-raw-rgb"))
45 create_element (const gchar * factory_name, GstElement ** element,
48 *element = gst_element_factory_make (factory_name, NULL);
52 if (err && *err == NULL) {
53 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
54 "cannot create element '%s' - please check your GStreamer installation",
62 get_encoder (const GstCaps * caps, GError ** err)
64 GList *encoders = NULL;
65 GList *filtered = NULL;
66 GstElementFactory *factory = NULL;
67 GstElement *encoder = NULL;
70 gst_element_factory_list_get_elements (GST_ELEMENT_FACTORY_TYPE_ENCODER |
71 GST_ELEMENT_FACTORY_TYPE_MEDIA_IMAGE, GST_RANK_NONE);
73 if (encoders == NULL) {
74 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
75 "Cannot find any image encoder");
79 GST_INFO ("got factory list %p", encoders);
80 gst_plugin_feature_list_debug (encoders);
83 gst_element_factory_list_filter (encoders, caps, GST_PAD_SRC, FALSE);
84 GST_INFO ("got filtered list %p", filtered);
86 if (filtered == NULL) {
87 gchar *tmp = gst_caps_to_string (caps);
88 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
89 "Cannot find any image encoder for caps %s", tmp);
94 gst_plugin_feature_list_debug (filtered);
96 factory = (GstElementFactory *) filtered->data;
98 GST_INFO ("got factory %p", factory);
99 encoder = gst_element_factory_create (factory, NULL);
101 GST_INFO ("created encoder element %p, %s", encoder,
102 gst_element_get_name (encoder));
106 gst_plugin_feature_list_free (encoders);
108 gst_plugin_feature_list_free (filtered);
114 build_convert_frame_pipeline (GstElement ** src_element,
115 GstElement ** sink_element, const GstCaps * from_caps,
116 const GstCaps * to_caps, GError ** err)
118 GstElement *src = NULL, *csp = NULL, *vscale = NULL;
119 GstElement *sink = NULL, *encoder = NULL, *pipeline;
120 GError *error = NULL;
122 /* videoscale is here to correct for the pixel-aspect-ratio for us */
123 GST_DEBUG ("creating elements");
124 if (!create_element ("appsrc", &src, &error) ||
125 !create_element ("ffmpegcolorspace", &csp, &error) ||
126 !create_element ("videoscale", &vscale, &error) ||
127 !create_element ("appsink", &sink, &error))
130 pipeline = gst_pipeline_new ("videoconvert-pipeline");
131 if (pipeline == NULL)
134 /* Add black borders if necessary to keep the DAR */
135 g_object_set (vscale, "add-borders", TRUE, NULL);
137 GST_DEBUG ("adding elements");
138 gst_bin_add_many (GST_BIN (pipeline), src, csp, vscale, sink, NULL);
141 g_object_set (src, "caps", from_caps, NULL);
142 g_object_set (sink, "caps", to_caps, NULL);
144 /* FIXME: linking is still way too expensive, profile this properly */
145 GST_DEBUG ("linking src->csp");
146 if (!gst_element_link_pads (src, "src", csp, "sink"))
149 GST_DEBUG ("linking csp->vscale");
150 if (!gst_element_link_pads (csp, "src", vscale, "sink"))
153 if (caps_are_raw (to_caps)) {
154 GST_DEBUG ("linking vscale->sink");
156 if (!gst_element_link_pads (vscale, "src", sink, "sink"))
159 encoder = get_encoder (to_caps, &error);
162 gst_bin_add (GST_BIN (pipeline), encoder);
164 GST_DEBUG ("linking vscale->encoder");
165 if (!gst_element_link (vscale, encoder))
168 GST_DEBUG ("linking encoder->sink");
169 if (!gst_element_link_pads (encoder, "src", sink, "sink"))
173 g_object_set (src, "emit-signals", TRUE, NULL);
174 g_object_set (sink, "emit-signals", TRUE, NULL);
177 *sink_element = sink;
183 gst_object_unref (pipeline);
185 GST_ERROR ("could not find an encoder for provided caps");
189 g_error_free (error);
196 gst_object_unref (src);
198 gst_object_unref (csp);
200 gst_object_unref (vscale);
202 gst_object_unref (sink);
203 GST_ERROR ("Could not convert video frame: %s", error->message);
207 g_error_free (error);
212 gst_object_unref (src);
213 gst_object_unref (csp);
214 gst_object_unref (vscale);
215 gst_object_unref (sink);
217 GST_ERROR ("Could not convert video frame: no pipeline (unknown error)");
219 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
220 "Could not convert video frame: no pipeline (unknown error)");
225 gst_object_unref (pipeline);
227 GST_ERROR ("Could not convert video frame: failed to link elements");
229 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_NEGOTIATION,
230 "Could not convert video frame: failed to link elements");
236 * gst_video_convert_frame:
238 * @to_caps: the #GstCaps to convert to
239 * @timeout: the maximum amount of time allowed for the processing.
240 * @error: pointer to a #GError. Can be %NULL.
242 * Converts a raw video buffer into the specified output caps.
244 * The output caps can be any raw video formats or any image formats (jpeg, png, ...).
246 * The width, height and pixel-aspect-ratio can also be specified in the output caps.
248 * Returns: The converted #GstBuffer, or %NULL if an error happened (in which case @err
249 * will point to the #GError).
255 gst_video_convert_frame (GstBuffer * buf, const GstCaps * to_caps,
256 GstClockTime timeout, GError ** error)
259 GstBuffer *result = NULL;
262 GstCaps *from_caps, *to_caps_copy = NULL;
264 GstElement *pipeline, *src, *sink;
267 g_return_val_if_fail (buf != NULL, NULL);
268 g_return_val_if_fail (to_caps != NULL, NULL);
269 g_return_val_if_fail (GST_BUFFER_CAPS (buf) != NULL, NULL);
271 from_caps = GST_BUFFER_CAPS (buf);
273 to_caps_copy = gst_caps_new_empty ();
274 n = gst_caps_get_size (to_caps);
275 for (i = 0; i < n; i++) {
276 GstStructure *s = gst_caps_get_structure (to_caps, i);
278 s = gst_structure_copy (s);
279 gst_structure_remove_field (s, "framerate");
280 gst_caps_append_structure (to_caps_copy, s);
284 build_convert_frame_pipeline (&src, &sink, from_caps, to_caps_copy, &err);
288 /* now set the pipeline to the paused state, after we push the buffer into
289 * appsrc, this should preroll the converted buffer in appsink */
290 GST_DEBUG ("running conversion pipeline to caps %" GST_PTR_FORMAT,
292 gst_element_set_state (pipeline, GST_STATE_PAUSED);
294 /* feed buffer in appsrc */
295 GST_DEBUG ("feeding buffer %p, size %u, caps %" GST_PTR_FORMAT,
296 buf, GST_BUFFER_SIZE (buf), from_caps);
297 g_signal_emit_by_name (src, "push-buffer", buf, &ret);
299 /* now see what happens. We either got an error somewhere or the pipeline
301 bus = gst_element_get_bus (pipeline);
302 msg = gst_bus_timed_pop_filtered (bus,
303 timeout, GST_MESSAGE_ERROR | GST_MESSAGE_ASYNC_DONE);
306 switch (GST_MESSAGE_TYPE (msg)) {
307 case GST_MESSAGE_ASYNC_DONE:
309 /* we're prerolled, get the frame from appsink */
310 g_signal_emit_by_name (sink, "pull-preroll", &result);
313 GST_DEBUG ("conversion successful: result = %p", result);
315 GST_ERROR ("prerolled but no result frame?!");
319 case GST_MESSAGE_ERROR:{
322 gst_message_parse_error (msg, &err, &dbg);
324 GST_ERROR ("Could not convert video frame: %s", err->message);
325 GST_DEBUG ("%s [debug: %s]", err->message, GST_STR_NULL (dbg));
335 g_return_val_if_reached (NULL);
338 gst_message_unref (msg);
340 GST_ERROR ("Could not convert video frame: timeout during conversion");
342 *error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
343 "Could not convert video frame: timeout during conversion");
346 gst_element_set_state (pipeline, GST_STATE_NULL);
347 gst_object_unref (bus);
348 gst_object_unref (pipeline);
349 gst_caps_unref (to_caps_copy);
356 gst_caps_unref (to_caps_copy);
370 GstElement *pipeline;
371 GstVideoConvertFrameCallback callback;
373 GDestroyNotify destroy_notify;
374 GMainContext *context;
378 } GstVideoConvertFrameContext;
382 GstVideoConvertFrameCallback callback;
386 GDestroyNotify destroy_notify;
388 GstVideoConvertFrameContext *context;
389 } GstVideoConvertFrameCallbackContext;
392 gst_video_convert_frame_context_free (GstVideoConvertFrameContext * ctx)
394 /* Wait until all users of the mutex are done */
395 g_mutex_lock (ctx->mutex);
396 g_mutex_unlock (ctx->mutex);
397 g_mutex_free (ctx->mutex);
399 g_source_remove (ctx->timeout_id);
401 gst_buffer_unref (ctx->buffer);
402 g_main_context_unref (ctx->context);
404 gst_element_set_state (ctx->pipeline, GST_STATE_NULL);
405 gst_object_unref (ctx->pipeline);
407 g_slice_free (GstVideoConvertFrameContext, ctx);
411 gst_video_convert_frame_callback_context_free
412 (GstVideoConvertFrameCallbackContext * ctx)
415 gst_video_convert_frame_context_free (ctx->context);
416 g_slice_free (GstVideoConvertFrameCallbackContext, ctx);
420 convert_frame_dispatch_callback (GstVideoConvertFrameCallbackContext * ctx)
422 ctx->callback (ctx->buffer, ctx->error, ctx->user_data);
424 if (ctx->destroy_notify)
425 ctx->destroy_notify (ctx->user_data);
431 convert_frame_finish (GstVideoConvertFrameContext * context, GstBuffer * buffer,
435 GstVideoConvertFrameCallbackContext *ctx;
437 if (context->timeout_id)
438 g_source_remove (context->timeout_id);
439 context->timeout_id = 0;
441 ctx = g_slice_new (GstVideoConvertFrameCallbackContext);
442 ctx->callback = context->callback;
443 ctx->user_data = context->user_data;
444 ctx->destroy_notify = context->destroy_notify;
445 ctx->buffer = buffer;
447 ctx->context = context;
449 source = g_timeout_source_new (0);
450 g_source_set_callback (source,
451 (GSourceFunc) convert_frame_dispatch_callback, ctx,
452 (GDestroyNotify) gst_video_convert_frame_callback_context_free);
453 g_source_attach (source, context->context);
454 g_source_unref (source);
456 context->finished = TRUE;
460 convert_frame_timeout_callback (GstVideoConvertFrameContext * context)
464 g_mutex_lock (context->mutex);
466 if (context->finished)
469 GST_ERROR ("Could not convert video frame: timeout");
471 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
472 "Could not convert video frame: timeout");
474 convert_frame_finish (context, NULL, error);
477 g_mutex_unlock (context->mutex);
482 convert_frame_bus_callback (GstBus * bus, GstMessage * message,
483 GstVideoConvertFrameContext * context)
485 g_mutex_lock (context->mutex);
487 if (context->finished)
490 switch (GST_MESSAGE_TYPE (message)) {
491 case GST_MESSAGE_ERROR:{
495 gst_message_parse_error (message, &error, &dbg);
497 GST_ERROR ("Could not convert video frame: %s", error->message);
498 GST_DEBUG ("%s [debug: %s]", error->message, GST_STR_NULL (dbg));
500 convert_frame_finish (context, NULL, error);
510 g_mutex_unlock (context->mutex);
516 convert_frame_need_data_callback (GstElement * src, guint size,
517 GstVideoConvertFrameContext * context)
519 GstFlowReturn ret = GST_FLOW_ERROR;
522 g_mutex_lock (context->mutex);
524 if (context->finished)
527 g_signal_emit_by_name (src, "push-buffer", context->buffer, &ret);
528 gst_buffer_unref (context->buffer);
529 context->buffer = NULL;
531 if (ret != GST_FLOW_OK) {
532 GST_ERROR ("Could not push video frame: %s", gst_flow_get_name (ret));
534 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
535 "Could not push video frame: %s", gst_flow_get_name (ret));
537 convert_frame_finish (context, NULL, error);
540 g_signal_handlers_disconnect_by_func (src, convert_frame_need_data_callback,
544 g_mutex_unlock (context->mutex);
548 convert_frame_new_buffer_callback (GstElement * sink,
549 GstVideoConvertFrameContext * context)
551 GstBuffer *buf = NULL;
552 GError *error = NULL;
554 g_mutex_lock (context->mutex);
556 if (context->finished)
559 g_signal_emit_by_name (sink, "pull-preroll", &buf);
562 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
563 "Could not get converted video frame");
566 convert_frame_finish (context, buf, error);
568 g_signal_handlers_disconnect_by_func (sink, convert_frame_need_data_callback,
572 g_mutex_unlock (context->mutex);
576 * gst_video_convert_frame_async:
578 * @to_caps: the #GstCaps to convert to
579 * @timeout: the maximum amount of time allowed for the processing.
580 * @callback: %GstVideoConvertFrameCallback that will be called after conversion.
581 * @user_data: extra data that will be passed to the @callback
582 * @destroy_notify: %GDestroyNotify to be called after @user_data is not needed anymore
584 * Converts a raw video buffer into the specified output caps.
586 * The output caps can be any raw video formats or any image formats (jpeg, png, ...).
588 * The width, height and pixel-aspect-ratio can also be specified in the output caps.
590 * @callback will be called after conversion, when an error occured or if conversion didn't
591 * finish after @timeout. @callback will always be called from the thread default
592 * %GMainContext, see g_main_context_get_thread_default(). If GLib before 2.22 is used,
593 * this will always be the global default main context.
595 * @destroy_notify will be called after the callback was called and @user_data is not needed
602 gst_video_convert_frame_async (GstBuffer * buf, const GstCaps * to_caps,
603 GstClockTime timeout, GstVideoConvertFrameCallback callback,
604 gpointer user_data, GDestroyNotify destroy_notify)
606 GMainContext *context = NULL;
607 GError *error = NULL;
609 GstCaps *from_caps, *to_caps_copy = NULL;
610 GstElement *pipeline, *src, *sink;
613 GstVideoConvertFrameContext *ctx;
615 g_return_if_fail (buf != NULL);
616 g_return_if_fail (to_caps != NULL);
617 g_return_if_fail (GST_BUFFER_CAPS (buf) != NULL);
618 g_return_if_fail (callback != NULL);
620 context = g_main_context_get_thread_default ();
623 context = g_main_context_default ();
625 from_caps = GST_BUFFER_CAPS (buf);
627 to_caps_copy = gst_caps_new_empty ();
628 n = gst_caps_get_size (to_caps);
629 for (i = 0; i < n; i++) {
630 GstStructure *s = gst_caps_get_structure (to_caps, i);
632 s = gst_structure_copy (s);
633 gst_structure_remove_field (s, "framerate");
634 gst_caps_append_structure (to_caps_copy, s);
638 build_convert_frame_pipeline (&src, &sink, from_caps, to_caps_copy,
643 bus = gst_element_get_bus (pipeline);
645 ctx = g_slice_new0 (GstVideoConvertFrameContext);
646 ctx->mutex = g_mutex_new ();
647 ctx->buffer = gst_buffer_ref (buf);
648 ctx->callback = callback;
649 ctx->user_data = user_data;
650 ctx->destroy_notify = destroy_notify;
651 ctx->context = g_main_context_ref (context);
652 ctx->finished = FALSE;
653 ctx->pipeline = pipeline;
655 if (timeout != GST_CLOCK_TIME_NONE) {
656 source = g_timeout_source_new (timeout / GST_MSECOND);
657 g_source_set_callback (source,
658 (GSourceFunc) convert_frame_timeout_callback, ctx, NULL);
659 ctx->timeout_id = g_source_attach (source, context);
660 g_source_unref (source);
663 g_signal_connect (src, "need-data",
664 G_CALLBACK (convert_frame_need_data_callback), ctx);
665 g_signal_connect (sink, "new-preroll",
666 G_CALLBACK (convert_frame_new_buffer_callback), ctx);
668 source = gst_bus_create_watch (bus);
669 g_source_set_callback (source, (GSourceFunc) convert_frame_bus_callback,
671 g_source_attach (source, context);
672 g_source_unref (source);
674 gst_element_set_state (pipeline, GST_STATE_PLAYING);
676 gst_object_unref (bus);
677 gst_caps_unref (to_caps_copy);
683 GstVideoConvertFrameCallbackContext *ctx;
686 gst_caps_unref (to_caps_copy);
688 ctx = g_slice_new0 (GstVideoConvertFrameCallbackContext);
689 ctx->callback = callback;
690 ctx->user_data = user_data;
691 ctx->destroy_notify = destroy_notify;
695 source = g_timeout_source_new (0);
696 g_source_set_callback (source,
697 (GSourceFunc) convert_frame_dispatch_callback, ctx,
698 (GDestroyNotify) gst_video_convert_frame_callback_context_free);
699 g_source_attach (source, context);
700 g_source_unref (source);