1 /* gstgoom.c: implementation of goom drawing element
2 * Copyright (C) <2001> Richard Boulton <richard@tartarus.org>
3 * (C) <2006> Wim Taymans <wim at fluendo dot com>
4 * (C) <2011> Wim Taymans <wim.taymans at gmail dot com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111-1307, USA.
23 * SECTION:element-goom
24 * @see_also: synaesthesia
26 * Goom is an audio visualisation element. It creates warping structures
27 * based on the incoming audio signal.
30 * <title>Example launch line</title>
32 * gst-launch -v audiotestsrc ! goom ! ffmpegcolorspace ! xvimagesink
44 #include <gst/video/video.h>
45 #include <gst/audio/audio.h>
52 GST_DEBUG_CATEGORY (goom_debug);
53 #define GST_CAT_DEFAULT goom_debug
55 #define DEFAULT_WIDTH 320
56 #define DEFAULT_HEIGHT 240
57 #define DEFAULT_FPS_N 25
58 #define DEFAULT_FPS_D 1
60 /* signals and args */
73 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
76 #if G_BYTE_ORDER == G_BIG_ENDIAN
77 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("xRGB"))
79 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("BGRx"))
83 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", /* the name of the pads */
84 GST_PAD_SINK, /* type of the pad */
85 GST_PAD_ALWAYS, /* ALWAYS/SOMETIMES */
86 GST_STATIC_CAPS ("audio/x-raw, "
87 "format = (string) " GST_AUDIO_NE (S16) ", "
88 "rate = (int) [ 8000, 96000 ], " "channels = (int) { 1, 2 }")
92 static void gst_goom_finalize (GObject * object);
94 static GstStateChangeReturn gst_goom_change_state (GstElement * element,
95 GstStateChange transition);
97 static GstFlowReturn gst_goom_chain (GstPad * pad, GstBuffer * buffer);
98 static gboolean gst_goom_src_event (GstPad * pad, GstEvent * event);
99 static gboolean gst_goom_sink_event (GstPad * pad, GstEvent * event);
101 static gboolean gst_goom_src_query (GstPad * pad, GstQuery * query);
103 #define gst_goom_parent_class parent_class
104 G_DEFINE_TYPE (GstGoom, gst_goom, GST_TYPE_ELEMENT);
107 gst_goom_class_init (GstGoomClass * klass)
109 GObjectClass *gobject_class;
110 GstElementClass *gstelement_class;
112 gobject_class = (GObjectClass *) klass;
113 gstelement_class = (GstElementClass *) klass;
115 gobject_class->finalize = gst_goom_finalize;
117 gst_element_class_set_details_simple (gstelement_class, "GOOM: what a GOOM!",
119 "Takes frames of data and outputs video frames using the GOOM filter",
120 "Wim Taymans <wim@fluendo.com>");
121 gst_element_class_add_pad_template (gstelement_class,
122 gst_static_pad_template_get (&sink_template));
123 gst_element_class_add_pad_template (gstelement_class,
124 gst_static_pad_template_get (&src_template));
126 gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_goom_change_state);
130 gst_goom_init (GstGoom * goom)
132 /* create the sink and src pads */
133 goom->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
134 gst_pad_set_chain_function (goom->sinkpad,
135 GST_DEBUG_FUNCPTR (gst_goom_chain));
136 gst_pad_set_event_function (goom->sinkpad,
137 GST_DEBUG_FUNCPTR (gst_goom_sink_event));
138 gst_element_add_pad (GST_ELEMENT (goom), goom->sinkpad);
140 goom->srcpad = gst_pad_new_from_static_template (&src_template, "src");
141 gst_pad_set_event_function (goom->srcpad,
142 GST_DEBUG_FUNCPTR (gst_goom_src_event));
143 gst_pad_set_query_function (goom->srcpad,
144 GST_DEBUG_FUNCPTR (gst_goom_src_query));
145 gst_element_add_pad (GST_ELEMENT (goom), goom->srcpad);
147 goom->adapter = gst_adapter_new ();
149 goom->width = DEFAULT_WIDTH;
150 goom->height = DEFAULT_HEIGHT;
151 goom->fps_n = DEFAULT_FPS_N; /* desired frame rate */
152 goom->fps_d = DEFAULT_FPS_D; /* desired frame rate */
157 goom->plugin = goom_init (goom->width, goom->height);
161 gst_goom_finalize (GObject * object)
163 GstGoom *goom = GST_GOOM (object);
165 goom_close (goom->plugin);
168 g_object_unref (goom->adapter);
170 gst_object_unref (goom->pool);
172 G_OBJECT_CLASS (parent_class)->finalize (object);
176 gst_goom_reset (GstGoom * goom)
178 gst_adapter_clear (goom->adapter);
179 gst_segment_init (&goom->segment, GST_FORMAT_UNDEFINED);
181 GST_OBJECT_LOCK (goom);
182 goom->proportion = 1.0;
183 goom->earliest_time = -1;
184 GST_OBJECT_UNLOCK (goom);
188 gst_goom_sink_setcaps (GstGoom * goom, GstCaps * caps)
190 GstStructure *structure;
192 structure = gst_caps_get_structure (caps, 0);
194 gst_structure_get_int (structure, "channels", &goom->channels);
195 gst_structure_get_int (structure, "rate", &goom->rate);
197 goom->bps = goom->channels * sizeof (gint16);
203 gst_goom_src_setcaps (GstGoom * goom, GstCaps * caps)
205 GstStructure *structure;
208 structure = gst_caps_get_structure (caps, 0);
209 if (!gst_structure_get_int (structure, "width", &goom->width) ||
210 !gst_structure_get_int (structure, "height", &goom->height) ||
211 !gst_structure_get_fraction (structure, "framerate", &goom->fps_n,
215 goom_set_resolution (goom->plugin, goom->width, goom->height);
217 /* size of the output buffer in bytes, depth is always 4 bytes */
218 goom->outsize = goom->width * goom->height * 4;
220 gst_util_uint64_scale_int (GST_SECOND, goom->fps_d, goom->fps_n);
221 goom->spf = gst_util_uint64_scale_int (goom->rate, goom->fps_d, goom->fps_n);
222 goom->bpf = goom->spf * goom->bps;
224 GST_DEBUG_OBJECT (goom, "dimension %dx%d, framerate %d/%d, spf %d",
225 goom->width, goom->height, goom->fps_n, goom->fps_d, goom->spf);
227 res = gst_pad_push_event (goom->srcpad, gst_event_new_caps (caps));
234 GST_DEBUG_OBJECT (goom, "error parsing caps");
240 gst_goom_src_negotiate (GstGoom * goom)
242 GstCaps *othercaps, *target;
243 GstStructure *structure;
246 GstBufferPool *pool = NULL;
247 guint size, min, max, prefix, alignment;
249 templ = gst_pad_get_pad_template_caps (goom->srcpad);
251 GST_DEBUG_OBJECT (goom, "performing negotiation");
253 /* see what the peer can do */
254 othercaps = gst_pad_peer_get_caps (goom->srcpad, NULL);
256 target = gst_caps_intersect (othercaps, templ);
257 gst_caps_unref (othercaps);
258 gst_caps_unref (templ);
260 if (gst_caps_is_empty (target))
263 gst_caps_truncate (target);
268 structure = gst_caps_get_structure (target, 0);
269 gst_structure_fixate_field_nearest_int (structure, "width", DEFAULT_WIDTH);
270 gst_structure_fixate_field_nearest_int (structure, "height", DEFAULT_HEIGHT);
271 gst_structure_fixate_field_nearest_fraction (structure, "framerate",
272 DEFAULT_FPS_N, DEFAULT_FPS_D);
274 gst_goom_src_setcaps (goom, target);
276 /* try to get a bufferpool now */
277 /* find a pool for the negotiated caps now */
278 query = gst_query_new_allocation (target, TRUE);
280 if (gst_pad_peer_query (goom->srcpad, query)) {
281 /* we got configuration from our peer, parse them */
282 gst_query_parse_allocation_params (query, &size, &min, &max, &prefix,
285 size = goom->outsize;
292 GstStructure *config;
294 /* we did not get a pool, make one ourselves then */
295 pool = gst_buffer_pool_new ();
297 config = gst_buffer_pool_get_config (pool);
298 gst_buffer_pool_config_set (config, target, size, min, max, prefix,
300 gst_buffer_pool_set_config (pool, config);
304 gst_object_unref (goom->pool);
308 gst_buffer_pool_set_active (pool, TRUE);
310 gst_caps_unref (target);
316 gst_caps_unref (target);
322 gst_goom_src_event (GstPad * pad, GstEvent * event)
327 goom = GST_GOOM (gst_pad_get_parent (pad));
329 switch (GST_EVENT_TYPE (event)) {
333 GstClockTimeDiff diff;
334 GstClockTime timestamp;
336 gst_event_parse_qos (event, NULL, &proportion, &diff, ×tamp);
338 /* save stuff for the _chain() function */
339 GST_OBJECT_LOCK (goom);
340 goom->proportion = proportion;
342 /* we're late, this is a good estimate for next displayable
343 * frame (see part-qos.txt) */
344 goom->earliest_time = timestamp + 2 * diff + goom->duration;
346 goom->earliest_time = timestamp + diff;
347 GST_OBJECT_UNLOCK (goom);
349 res = gst_pad_push_event (goom->sinkpad, event);
353 res = gst_pad_push_event (goom->sinkpad, event);
356 gst_object_unref (goom);
362 gst_goom_sink_event (GstPad * pad, GstEvent * event)
367 goom = GST_GOOM (gst_pad_get_parent (pad));
369 switch (GST_EVENT_TYPE (event)) {
374 gst_event_parse_caps (event, &caps);
375 res = gst_goom_sink_setcaps (goom, caps);
378 case GST_EVENT_FLUSH_START:
379 res = gst_pad_push_event (goom->srcpad, event);
381 case GST_EVENT_FLUSH_STOP:
382 gst_goom_reset (goom);
383 res = gst_pad_push_event (goom->srcpad, event);
385 case GST_EVENT_SEGMENT:
387 /* the newsegment values are used to clip the input samples
388 * and to convert the incomming timestamps to running time so
390 gst_event_copy_segment (event, &goom->segment);
392 res = gst_pad_push_event (goom->srcpad, event);
396 res = gst_pad_push_event (goom->srcpad, event);
399 gst_object_unref (goom);
405 gst_goom_src_query (GstPad * pad, GstQuery * query)
407 gboolean res = FALSE;
410 goom = GST_GOOM (gst_pad_get_parent (pad));
412 switch (GST_QUERY_TYPE (query)) {
413 case GST_QUERY_LATENCY:
415 /* We need to send the query upstream and add the returned latency to our
417 GstClockTime min_latency, max_latency;
419 GstClockTime our_latency;
425 if ((res = gst_pad_peer_query (goom->sinkpad, query))) {
426 gst_query_parse_latency (query, &us_live, &min_latency, &max_latency);
428 GST_DEBUG_OBJECT (goom, "Peer latency: min %"
429 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
430 GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
432 /* the max samples we must buffer buffer */
433 max_samples = MAX (GOOM_SAMPLES, goom->spf);
435 gst_util_uint64_scale_int (max_samples, GST_SECOND, goom->rate);
437 GST_DEBUG_OBJECT (goom, "Our latency: %" GST_TIME_FORMAT,
438 GST_TIME_ARGS (our_latency));
440 /* we add some latency but only if we need to buffer more than what
441 * upstream gives us */
442 min_latency += our_latency;
443 if (max_latency != -1)
444 max_latency += our_latency;
446 GST_DEBUG_OBJECT (goom, "Calculated total latency : min %"
447 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
448 GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
450 gst_query_set_latency (query, TRUE, min_latency, max_latency);
455 res = gst_pad_peer_query (goom->sinkpad, query);
459 gst_object_unref (goom);
464 /* make sure we are negotiated */
466 ensure_negotiated (GstGoom * goom)
468 gboolean reconfigure;
470 GST_OBJECT_LOCK (goom->srcpad);
471 reconfigure = GST_PAD_NEEDS_RECONFIGURE (goom->srcpad);
472 GST_OBJECT_FLAG_UNSET (goom->srcpad, GST_PAD_NEED_RECONFIGURE);
473 GST_OBJECT_UNLOCK (goom->srcpad);
475 /* we don't know an output format yet, pick one */
476 if (reconfigure || !gst_pad_has_current_caps (goom->srcpad)) {
477 if (!gst_goom_src_negotiate (goom))
478 return GST_FLOW_NOT_NEGOTIATED;
485 gst_goom_chain (GstPad * pad, GstBuffer * buffer)
489 GstBuffer *outbuf = NULL;
491 goom = GST_GOOM (gst_pad_get_parent (pad));
492 if (goom->bps == 0) {
493 ret = GST_FLOW_NOT_NEGOTIATED;
497 /* Make sure have an output format */
498 ret = ensure_negotiated (goom);
499 if (ret != GST_FLOW_OK) {
500 gst_buffer_unref (buffer);
504 /* don't try to combine samples from discont buffer */
505 if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) {
506 gst_adapter_clear (goom->adapter);
509 GST_DEBUG_OBJECT (goom,
510 "Input buffer has %d samples, time=%" G_GUINT64_FORMAT,
511 gst_buffer_get_size (buffer) / goom->bps, GST_BUFFER_TIMESTAMP (buffer));
513 /* Collect samples until we have enough for an output frame */
514 gst_adapter_push (goom->adapter, buffer);
523 guint avail, to_flush;
524 guint64 dist, timestamp;
526 avail = gst_adapter_available (goom->adapter);
527 GST_DEBUG_OBJECT (goom, "avail now %u", avail);
529 /* we need GOOM_SAMPLES to get a meaningful result from goom. */
530 if (avail < (GOOM_SAMPLES * goom->bps))
533 /* we also need enough samples to produce one frame at least */
534 if (avail < goom->bpf)
537 GST_DEBUG_OBJECT (goom, "processing buffer");
539 /* get timestamp of the current adapter byte */
540 timestamp = gst_adapter_prev_timestamp (goom->adapter, &dist);
541 if (GST_CLOCK_TIME_IS_VALID (timestamp)) {
542 /* convert bytes to time */
544 timestamp += gst_util_uint64_scale_int (dist, GST_SECOND, goom->rate);
547 if (timestamp != -1) {
550 qostime = gst_segment_to_running_time (&goom->segment, GST_FORMAT_TIME,
552 qostime += goom->duration;
554 GST_OBJECT_LOCK (goom);
555 /* check for QoS, don't compute buffers that are known to be late */
556 need_skip = goom->earliest_time != -1 && qostime <= goom->earliest_time;
557 GST_OBJECT_UNLOCK (goom);
560 GST_WARNING_OBJECT (goom,
561 "QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT,
562 GST_TIME_ARGS (qostime), GST_TIME_ARGS (goom->earliest_time));
567 /* get next GOOM_SAMPLES, we have at least this amount of samples */
569 (const guint16 *) gst_adapter_map (goom->adapter,
570 GOOM_SAMPLES * goom->bps);
572 if (goom->channels == 2) {
573 for (i = 0; i < GOOM_SAMPLES; i++) {
574 goom->datain[0][i] = *data++;
575 goom->datain[1][i] = *data++;
578 for (i = 0; i < GOOM_SAMPLES; i++) {
579 goom->datain[0][i] = *data;
580 goom->datain[1][i] = *data++;
584 /* alloc a buffer if we don't have one yet, this happens
585 * when we pushed a buffer in this while loop before */
586 if (outbuf == NULL) {
587 GST_DEBUG_OBJECT (goom, "allocating output buffer");
588 ret = gst_buffer_pool_acquire_buffer (goom->pool, &outbuf, NULL);
589 if (ret != GST_FLOW_OK) {
590 gst_adapter_unmap (goom->adapter, 0);
595 GST_BUFFER_TIMESTAMP (outbuf) = timestamp;
596 GST_BUFFER_DURATION (outbuf) = goom->duration;
598 out_frame = (guchar *) goom_update (goom->plugin, goom->datain, 0, 0);
599 gst_buffer_fill (outbuf, 0, out_frame, goom->outsize);
601 gst_adapter_unmap (goom->adapter, 0);
603 GST_DEBUG ("Pushing frame with time=%" GST_TIME_FORMAT ", duration=%"
604 GST_TIME_FORMAT, GST_TIME_ARGS (timestamp),
605 GST_TIME_ARGS (goom->duration));
607 ret = gst_pad_push (goom->srcpad, outbuf);
611 /* Now flush the samples we needed for this frame, which might be more than
612 * the samples we used (GOOM_SAMPLES). */
613 to_flush = goom->bpf;
615 GST_DEBUG_OBJECT (goom, "finished frame, flushing %u bytes from input",
617 gst_adapter_flush (goom->adapter, to_flush);
619 if (ret != GST_FLOW_OK)
624 gst_buffer_unref (outbuf);
627 gst_object_unref (goom);
632 static GstStateChangeReturn
633 gst_goom_change_state (GstElement * element, GstStateChange transition)
635 GstGoom *goom = GST_GOOM (element);
636 GstStateChangeReturn ret;
638 switch (transition) {
639 case GST_STATE_CHANGE_NULL_TO_READY:
641 case GST_STATE_CHANGE_READY_TO_PAUSED:
642 gst_goom_reset (goom);
648 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
650 switch (transition) {
651 case GST_STATE_CHANGE_PAUSED_TO_READY:
653 gst_buffer_pool_set_active (goom->pool, FALSE);
654 gst_object_unref (goom->pool);
658 case GST_STATE_CHANGE_READY_TO_NULL:
668 plugin_init (GstPlugin * plugin)
670 GST_DEBUG_CATEGORY_INIT (goom_debug, "goom", 0, "goom visualisation element");
676 return gst_element_register (plugin, "goom", GST_RANK_NONE, GST_TYPE_GOOM);
679 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
682 "GOOM visualization filter",
683 plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)