1 /* Small helper element for format conversion
2 * Copyright (C) 2005 Tim-Philipp Müller <tim centricular net>
3 * Copyright (C) 2010 Brandon Lewis <brandon.lewis@collabora.co.uk>
4 * Copyright (C) 2010 Edward Hervey <edward.hervey@collabora.co.uk>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
25 #include "gst/glib-compat-private.h"
28 caps_are_raw (const GstCaps * caps)
32 len = gst_caps_get_size (caps);
34 for (i = 0; i < len; i++) {
35 GstStructure *st = gst_caps_get_structure (caps, i);
36 if (gst_structure_has_name (st, "video/x-raw"))
44 create_element (const gchar * factory_name, GstElement ** element,
47 *element = gst_element_factory_make (factory_name, NULL);
51 if (err && *err == NULL) {
52 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
53 "cannot create element '%s' - please check your GStreamer installation",
61 get_encoder (const GstCaps * caps, GError ** err)
63 GList *encoders = NULL;
64 GList *filtered = NULL;
65 GstElementFactory *factory = NULL;
66 GstElement *encoder = NULL;
69 gst_element_factory_list_get_elements (GST_ELEMENT_FACTORY_TYPE_ENCODER |
70 GST_ELEMENT_FACTORY_TYPE_MEDIA_IMAGE, GST_RANK_NONE);
72 if (encoders == NULL) {
73 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
74 "Cannot find any image encoder");
78 GST_INFO ("got factory list %p", encoders);
79 gst_plugin_feature_list_debug (encoders);
82 gst_element_factory_list_filter (encoders, caps, GST_PAD_SRC, FALSE);
83 GST_INFO ("got filtered list %p", filtered);
85 if (filtered == NULL) {
86 gchar *tmp = gst_caps_to_string (caps);
87 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
88 "Cannot find any image encoder for caps %s", tmp);
93 gst_plugin_feature_list_debug (filtered);
95 factory = (GstElementFactory *) filtered->data;
97 GST_INFO ("got factory %p", factory);
98 encoder = gst_element_factory_create (factory, NULL);
100 GST_INFO ("created encoder element %p, %s", encoder,
101 gst_element_get_name (encoder));
105 gst_plugin_feature_list_free (encoders);
107 gst_plugin_feature_list_free (filtered);
113 build_convert_frame_pipeline (GstElement ** src_element,
114 GstElement ** sink_element, const GstCaps * from_caps,
115 const GstCaps * to_caps, GError ** err)
117 GstElement *src = NULL, *csp = NULL, *vscale = NULL;
118 GstElement *sink = NULL, *encoder = NULL, *pipeline;
119 GError *error = NULL;
121 /* videoscale is here to correct for the pixel-aspect-ratio for us */
122 GST_DEBUG ("creating elements");
123 if (!create_element ("appsrc", &src, &error) ||
124 !create_element ("videoconvert", &csp, &error) ||
125 !create_element ("videoscale", &vscale, &error) ||
126 !create_element ("appsink", &sink, &error))
129 pipeline = gst_pipeline_new ("videoconvert-pipeline");
130 if (pipeline == NULL)
133 /* Add black borders if necessary to keep the DAR */
134 g_object_set (vscale, "add-borders", TRUE, NULL);
136 GST_DEBUG ("adding elements");
137 gst_bin_add_many (GST_BIN (pipeline), src, csp, vscale, sink, NULL);
140 g_object_set (src, "caps", from_caps, NULL);
141 g_object_set (sink, "caps", to_caps, NULL);
143 /* FIXME: linking is still way too expensive, profile this properly */
144 GST_DEBUG ("linking src->csp");
145 if (!gst_element_link_pads (src, "src", csp, "sink"))
148 GST_DEBUG ("linking csp->vscale");
149 if (!gst_element_link_pads_full (csp, "src", vscale, "sink",
150 GST_PAD_LINK_CHECK_NOTHING))
153 if (caps_are_raw (to_caps)) {
154 GST_DEBUG ("linking vscale->sink");
156 if (!gst_element_link_pads_full (vscale, "src", sink, "sink",
157 GST_PAD_LINK_CHECK_NOTHING))
160 encoder = get_encoder (to_caps, &error);
163 gst_bin_add (GST_BIN (pipeline), encoder);
165 GST_DEBUG ("linking vscale->encoder");
166 if (!gst_element_link (vscale, encoder))
169 GST_DEBUG ("linking encoder->sink");
170 if (!gst_element_link_pads (encoder, "src", sink, "sink"))
174 g_object_set (src, "emit-signals", TRUE, NULL);
175 g_object_set (sink, "emit-signals", TRUE, NULL);
178 *sink_element = sink;
184 gst_object_unref (pipeline);
186 GST_ERROR ("could not find an encoder for provided caps");
190 g_error_free (error);
197 gst_object_unref (src);
199 gst_object_unref (csp);
201 gst_object_unref (vscale);
203 gst_object_unref (sink);
204 GST_ERROR ("Could not convert video frame: %s", error->message);
208 g_error_free (error);
213 gst_object_unref (src);
214 gst_object_unref (csp);
215 gst_object_unref (vscale);
216 gst_object_unref (sink);
218 GST_ERROR ("Could not convert video frame: no pipeline (unknown error)");
220 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
221 "Could not convert video frame: no pipeline (unknown error)");
226 gst_object_unref (pipeline);
228 GST_ERROR ("Could not convert video frame: failed to link elements");
230 *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_NEGOTIATION,
231 "Could not convert video frame: failed to link elements");
237 * gst_video_convert_sample:
238 * @sample: a #GstSample
239 * @to_caps: the #GstCaps to convert to
240 * @timeout: the maximum amount of time allowed for the processing.
241 * @error: pointer to a #GError. Can be %NULL.
243 * Converts a raw video buffer into the specified output caps.
245 * The output caps can be any raw video formats or any image formats (jpeg, png, ...).
247 * The width, height and pixel-aspect-ratio can also be specified in the output caps.
249 * Returns: The converted #GstSample, or %NULL if an error happened (in which case @err
250 * will point to the #GError).
256 gst_video_convert_sample (GstSample * sample, const GstCaps * to_caps,
257 GstClockTime timeout, GError ** error)
261 GstSample *result = NULL;
264 GstCaps *from_caps, *to_caps_copy = NULL;
266 GstElement *pipeline, *src, *sink;
269 g_return_val_if_fail (sample != NULL, NULL);
270 g_return_val_if_fail (to_caps != NULL, NULL);
272 buf = gst_sample_get_buffer (sample);
273 g_return_val_if_fail (buf != NULL, NULL);
275 from_caps = gst_sample_get_caps (sample);
276 g_return_val_if_fail (from_caps != NULL, NULL);
279 to_caps_copy = gst_caps_new_empty ();
280 n = gst_caps_get_size (to_caps);
281 for (i = 0; i < n; i++) {
282 GstStructure *s = gst_caps_get_structure (to_caps, i);
284 s = gst_structure_copy (s);
285 gst_structure_remove_field (s, "framerate");
286 gst_caps_append_structure (to_caps_copy, s);
290 build_convert_frame_pipeline (&src, &sink, from_caps, to_caps_copy, &err);
294 /* now set the pipeline to the paused state, after we push the buffer into
295 * appsrc, this should preroll the converted buffer in appsink */
296 GST_DEBUG ("running conversion pipeline to caps %" GST_PTR_FORMAT,
298 gst_element_set_state (pipeline, GST_STATE_PAUSED);
300 /* feed buffer in appsrc */
301 GST_DEBUG ("feeding buffer %p, size %" G_GSIZE_FORMAT ", caps %"
302 GST_PTR_FORMAT, buf, gst_buffer_get_size (buf), from_caps);
303 g_signal_emit_by_name (src, "push-buffer", buf, &ret);
305 /* now see what happens. We either got an error somewhere or the pipeline
307 bus = gst_element_get_bus (pipeline);
308 msg = gst_bus_timed_pop_filtered (bus,
309 timeout, GST_MESSAGE_ERROR | GST_MESSAGE_ASYNC_DONE);
312 switch (GST_MESSAGE_TYPE (msg)) {
313 case GST_MESSAGE_ASYNC_DONE:
315 /* we're prerolled, get the frame from appsink */
316 g_signal_emit_by_name (sink, "pull-preroll", &result);
319 GST_DEBUG ("conversion successful: result = %p", result);
321 GST_ERROR ("prerolled but no result frame?!");
325 case GST_MESSAGE_ERROR:{
328 gst_message_parse_error (msg, &err, &dbg);
330 GST_ERROR ("Could not convert video frame: %s", err->message);
331 GST_DEBUG ("%s [debug: %s]", err->message, GST_STR_NULL (dbg));
341 g_return_val_if_reached (NULL);
344 gst_message_unref (msg);
346 GST_ERROR ("Could not convert video frame: timeout during conversion");
348 *error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
349 "Could not convert video frame: timeout during conversion");
352 gst_element_set_state (pipeline, GST_STATE_NULL);
353 gst_object_unref (bus);
354 gst_object_unref (pipeline);
355 gst_caps_unref (to_caps_copy);
362 gst_caps_unref (to_caps_copy);
376 GstElement *pipeline;
377 GstVideoConvertSampleCallback callback;
379 GDestroyNotify destroy_notify;
380 GMainContext *context;
385 } GstVideoConvertSampleContext;
389 GstVideoConvertSampleCallback callback;
394 GDestroyNotify destroy_notify;
396 GstVideoConvertSampleContext *context;
397 } GstVideoConvertSampleCallbackContext;
400 gst_video_convert_frame_context_free (GstVideoConvertSampleContext * ctx)
402 /* Wait until all users of the mutex are done */
403 g_mutex_lock (ctx->mutex);
404 g_mutex_unlock (ctx->mutex);
405 g_mutex_free (ctx->mutex);
407 g_source_remove (ctx->timeout_id);
409 // gst_buffer_unref (ctx->buffer);
411 gst_sample_unref (ctx->sample);
412 g_main_context_unref (ctx->context);
414 gst_element_set_state (ctx->pipeline, GST_STATE_NULL);
415 gst_object_unref (ctx->pipeline);
417 g_slice_free (GstVideoConvertSampleContext, ctx);
421 gst_video_convert_frame_callback_context_free
422 (GstVideoConvertSampleCallbackContext * ctx)
425 gst_video_convert_frame_context_free (ctx->context);
426 g_slice_free (GstVideoConvertSampleCallbackContext, ctx);
430 convert_frame_dispatch_callback (GstVideoConvertSampleCallbackContext * ctx)
432 ctx->callback (ctx->sample, ctx->error, ctx->user_data);
434 if (ctx->destroy_notify)
435 ctx->destroy_notify (ctx->user_data);
441 convert_frame_finish (GstVideoConvertSampleContext * context,
442 GstSample * sample, GError * error)
445 GstVideoConvertSampleCallbackContext *ctx;
447 if (context->timeout_id)
448 g_source_remove (context->timeout_id);
449 context->timeout_id = 0;
451 ctx = g_slice_new (GstVideoConvertSampleCallbackContext);
452 ctx->callback = context->callback;
453 ctx->user_data = context->user_data;
454 ctx->destroy_notify = context->destroy_notify;
455 ctx->sample = sample;
456 //ctx->buffer = buffer;
458 ctx->context = context;
460 source = g_timeout_source_new (0);
461 g_source_set_callback (source,
462 (GSourceFunc) convert_frame_dispatch_callback, ctx,
463 (GDestroyNotify) gst_video_convert_frame_callback_context_free);
464 g_source_attach (source, context->context);
465 g_source_unref (source);
467 context->finished = TRUE;
471 convert_frame_timeout_callback (GstVideoConvertSampleContext * context)
475 g_mutex_lock (context->mutex);
477 if (context->finished)
480 GST_ERROR ("Could not convert video frame: timeout");
482 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
483 "Could not convert video frame: timeout");
485 convert_frame_finish (context, NULL, error);
488 g_mutex_unlock (context->mutex);
493 convert_frame_bus_callback (GstBus * bus, GstMessage * message,
494 GstVideoConvertSampleContext * context)
496 g_mutex_lock (context->mutex);
498 if (context->finished)
501 switch (GST_MESSAGE_TYPE (message)) {
502 case GST_MESSAGE_ERROR:{
506 gst_message_parse_error (message, &error, &dbg);
508 GST_ERROR ("Could not convert video frame: %s", error->message);
509 GST_DEBUG ("%s [debug: %s]", error->message, GST_STR_NULL (dbg));
511 convert_frame_finish (context, NULL, error);
521 g_mutex_unlock (context->mutex);
527 convert_frame_need_data_callback (GstElement * src, guint size,
528 GstVideoConvertSampleContext * context)
530 GstFlowReturn ret = GST_FLOW_ERROR;
534 g_mutex_lock (context->mutex);
536 if (context->finished)
539 buffer = gst_sample_get_buffer (context->sample);
540 g_signal_emit_by_name (src, "push-buffer", buffer, &ret);
541 gst_sample_unref (context->sample);
542 context->sample = NULL;
544 if (ret != GST_FLOW_OK) {
545 GST_ERROR ("Could not push video frame: %s", gst_flow_get_name (ret));
547 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
548 "Could not push video frame: %s", gst_flow_get_name (ret));
550 convert_frame_finish (context, NULL, error);
553 g_signal_handlers_disconnect_by_func (src, convert_frame_need_data_callback,
557 g_mutex_unlock (context->mutex);
561 convert_frame_new_preroll_callback (GstElement * sink,
562 GstVideoConvertSampleContext * context)
564 GstSample *sample = NULL;
565 GError *error = NULL;
567 g_mutex_lock (context->mutex);
569 if (context->finished)
572 g_signal_emit_by_name (sink, "pull-preroll", &sample);
575 error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
576 "Could not get converted video sample");
578 convert_frame_finish (context, sample, error);
580 g_signal_handlers_disconnect_by_func (sink, convert_frame_need_data_callback,
584 g_mutex_unlock (context->mutex);
588 * gst_video_convert_sample_async:
589 * @sample: a #GstSample
590 * @to_caps: the #GstCaps to convert to
591 * @timeout: the maximum amount of time allowed for the processing.
592 * @callback: %GstVideoConvertSampleCallback that will be called after conversion.
593 * @user_data: extra data that will be passed to the @callback
594 * @destroy_notify: %GDestroyNotify to be called after @user_data is not needed anymore
596 * Converts a raw video buffer into the specified output caps.
598 * The output caps can be any raw video formats or any image formats (jpeg, png, ...).
600 * The width, height and pixel-aspect-ratio can also be specified in the output caps.
602 * @callback will be called after conversion, when an error occured or if conversion didn't
603 * finish after @timeout. @callback will always be called from the thread default
604 * %GMainContext, see g_main_context_get_thread_default(). If GLib before 2.22 is used,
605 * this will always be the global default main context.
607 * @destroy_notify will be called after the callback was called and @user_data is not needed
614 gst_video_convert_sample_async (GstSample * sample,
615 const GstCaps * to_caps, GstClockTime timeout,
616 GstVideoConvertSampleCallback callback, gpointer user_data,
617 GDestroyNotify destroy_notify)
619 GMainContext *context = NULL;
620 GError *error = NULL;
623 GstCaps *from_caps, *to_caps_copy = NULL;
624 GstElement *pipeline, *src, *sink;
627 GstVideoConvertSampleContext *ctx;
629 g_return_if_fail (sample != NULL);
630 buf = gst_sample_get_buffer (sample);
631 g_return_if_fail (buf != NULL);
633 g_return_if_fail (to_caps != NULL);
635 from_caps = gst_sample_get_caps (sample);
636 g_return_if_fail (from_caps != NULL);
637 g_return_if_fail (callback != NULL);
639 context = g_main_context_get_thread_default ();
642 context = g_main_context_default ();
644 to_caps_copy = gst_caps_new_empty ();
645 n = gst_caps_get_size (to_caps);
646 for (i = 0; i < n; i++) {
647 GstStructure *s = gst_caps_get_structure (to_caps, i);
649 s = gst_structure_copy (s);
650 gst_structure_remove_field (s, "framerate");
651 gst_caps_append_structure (to_caps_copy, s);
655 build_convert_frame_pipeline (&src, &sink, from_caps, to_caps_copy,
660 bus = gst_element_get_bus (pipeline);
662 ctx = g_slice_new0 (GstVideoConvertSampleContext);
663 ctx->mutex = g_mutex_new ();
664 //ctx->buffer = gst_buffer_ref (buf);
665 ctx->sample = gst_sample_ref (sample);
666 ctx->callback = callback;
667 ctx->user_data = user_data;
668 ctx->destroy_notify = destroy_notify;
669 ctx->context = g_main_context_ref (context);
670 ctx->finished = FALSE;
671 ctx->pipeline = pipeline;
673 if (timeout != GST_CLOCK_TIME_NONE) {
674 source = g_timeout_source_new (timeout / GST_MSECOND);
675 g_source_set_callback (source,
676 (GSourceFunc) convert_frame_timeout_callback, ctx, NULL);
677 ctx->timeout_id = g_source_attach (source, context);
678 g_source_unref (source);
681 g_signal_connect (src, "need-data",
682 G_CALLBACK (convert_frame_need_data_callback), ctx);
683 g_signal_connect (sink, "new-preroll",
684 G_CALLBACK (convert_frame_new_preroll_callback), ctx);
686 source = gst_bus_create_watch (bus);
687 g_source_set_callback (source, (GSourceFunc) convert_frame_bus_callback,
689 g_source_attach (source, context);
690 g_source_unref (source);
692 gst_element_set_state (pipeline, GST_STATE_PLAYING);
694 gst_object_unref (bus);
695 gst_caps_unref (to_caps_copy);
701 GstVideoConvertSampleCallbackContext *ctx;
704 gst_caps_unref (to_caps_copy);
706 ctx = g_slice_new0 (GstVideoConvertSampleCallbackContext);
707 ctx->callback = callback;
708 ctx->user_data = user_data;
709 ctx->destroy_notify = destroy_notify;
713 source = g_timeout_source_new (0);
714 g_source_set_callback (source,
715 (GSourceFunc) convert_frame_dispatch_callback, ctx,
716 (GDestroyNotify) gst_video_convert_frame_callback_context_free);
717 g_source_attach (source, context);
718 g_source_unref (source);