2 * Copyright (C) <2011> Stefan Kost <ensonic@users.sf.net>
3 * Copyright (C) <2015> Luis de Bethencourt <luis@debethencourt.com>
5 * gstaudiovisualizer.h: base class for audio visualisation elements
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
23 * SECTION:gstaudiovisualizer
24 * @title: GstAudioVisualizer
25 * @short_description: Base class for visualizers.
27 * A baseclass for scopes (visualizers). It takes care of re-fitting the
28 * audio-rate to video-rate and handles renegotiation (downstream video size
31 * It also provides several background shading effects. These effects are
32 * applied to a previous picture before the `render()` implementation can draw a
42 #include <gst/video/video.h>
43 #include <gst/video/gstvideometa.h>
44 #include <gst/video/gstvideopool.h>
46 #include "gstaudiovisualizer.h"
47 #include "pbutils-enumtypes.h"
49 GST_DEBUG_CATEGORY_STATIC (audio_visualizer_debug);
50 #define GST_CAT_DEFAULT (audio_visualizer_debug)
52 #define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE
53 #define DEFAULT_SHADE_AMOUNT 0x000a0a0a
62 static GstBaseTransformClass *parent_class = NULL;
63 static gint private_offset = 0;
65 static void gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass);
66 static void gst_audio_visualizer_init (GstAudioVisualizer * scope,
67 GstAudioVisualizerClass * g_class);
68 static void gst_audio_visualizer_set_property (GObject * object,
69 guint prop_id, const GValue * value, GParamSpec * pspec);
70 static void gst_audio_visualizer_get_property (GObject * object,
71 guint prop_id, GValue * value, GParamSpec * pspec);
72 static void gst_audio_visualizer_dispose (GObject * object);
74 static gboolean gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope);
75 static gboolean gst_audio_visualizer_src_setcaps (GstAudioVisualizer *
76 scope, GstCaps * caps);
77 static gboolean gst_audio_visualizer_sink_setcaps (GstAudioVisualizer *
78 scope, GstCaps * caps);
80 static GstFlowReturn gst_audio_visualizer_chain (GstPad * pad,
81 GstObject * parent, GstBuffer * buffer);
83 static gboolean gst_audio_visualizer_src_event (GstPad * pad,
84 GstObject * parent, GstEvent * event);
85 static gboolean gst_audio_visualizer_sink_event (GstPad * pad,
86 GstObject * parent, GstEvent * event);
88 static gboolean gst_audio_visualizer_src_query (GstPad * pad,
89 GstObject * parent, GstQuery * query);
91 static GstStateChangeReturn gst_audio_visualizer_change_state (GstElement *
92 element, GstStateChange transition);
94 static gboolean gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
98 default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query);
100 struct _GstAudioVisualizerPrivate
105 gboolean pool_active;
106 GstAllocator *allocator;
107 GstAllocationParams params;
111 GstPad *srcpad, *sinkpad;
113 GstAudioVisualizerShader shader_type;
114 GstAudioVisualizerShaderFunc shader;
115 guint32 shade_amount;
121 GstVideoFrame tempframe;
123 guint spf; /* samples per video frame */
124 guint64 frame_duration;
126 /* QoS stuff *//* with LOCK */
128 GstClockTime earliest_time;
130 guint dropped; /* frames dropped / not dropped */
133 /* configuration mutex */
139 /* shading functions */
141 /* we're only supporting GST_VIDEO_FORMAT_xRGB right now) */
142 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
144 #define SHADE(_d, _s, _i, _r, _g, _b) \
146 _d[_i * 4 + 0] = (_s[_i * 4 + 0] > _b) ? _s[_i * 4 + 0] - _b : 0; \
147 _d[_i * 4 + 1] = (_s[_i * 4 + 1] > _g) ? _s[_i * 4 + 1] - _g : 0; \
148 _d[_i * 4 + 2] = (_s[_i * 4 + 2] > _r) ? _s[_i * 4 + 2] - _r : 0; \
149 _d[_i * 4 + 3] = 0; \
152 #else /* G_BYTE_ORDER == G_LITTLE_ENDIAN */
154 #define SHADE(_d, _s, _i, _r, _g, _b) \
156 _d[_i * 4 + 0] = 0; \
157 _d[_i * 4 + 1] = (_s[_i * 4 + 1] > _r) ? _s[_i * 4 + 1] - _r : 0; \
158 _d[_i * 4 + 2] = (_s[_i * 4 + 2] > _g) ? _s[_i * 4 + 2] - _g : 0; \
159 _d[_i * 4 + 3] = (_s[_i * 4 + 3] > _b) ? _s[_i * 4 + 3] - _b : 0; \
165 shader_fade (GstAudioVisualizer * scope, const GstVideoFrame * sframe,
166 GstVideoFrame * dframe)
169 guint r = (scope->priv->shade_amount >> 16) & 0xff;
170 guint g = (scope->priv->shade_amount >> 8) & 0xff;
171 guint b = (scope->priv->shade_amount >> 0) & 0xff;
173 gint ss, ds, width, height;
175 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
176 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
177 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
178 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
180 width = GST_VIDEO_FRAME_WIDTH (sframe);
181 height = GST_VIDEO_FRAME_HEIGHT (sframe);
183 for (j = 0; j < height; j++) {
184 for (i = 0; i < width; i++) {
185 SHADE (d, s, i, r, g, b);
193 shader_fade_and_move_up (GstAudioVisualizer * scope,
194 const GstVideoFrame * sframe, GstVideoFrame * dframe)
197 guint r = (scope->priv->shade_amount >> 16) & 0xff;
198 guint g = (scope->priv->shade_amount >> 8) & 0xff;
199 guint b = (scope->priv->shade_amount >> 0) & 0xff;
201 gint ss, ds, width, height;
203 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
204 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
205 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
206 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
208 width = GST_VIDEO_FRAME_WIDTH (sframe);
209 height = GST_VIDEO_FRAME_HEIGHT (sframe);
211 for (j = 1; j < height; j++) {
213 for (i = 0; i < width; i++) {
214 SHADE (d, s, i, r, g, b);
221 shader_fade_and_move_down (GstAudioVisualizer * scope,
222 const GstVideoFrame * sframe, GstVideoFrame * dframe)
225 guint r = (scope->priv->shade_amount >> 16) & 0xff;
226 guint g = (scope->priv->shade_amount >> 8) & 0xff;
227 guint b = (scope->priv->shade_amount >> 0) & 0xff;
229 gint ss, ds, width, height;
231 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
232 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
233 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
234 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
236 width = GST_VIDEO_FRAME_WIDTH (sframe);
237 height = GST_VIDEO_FRAME_HEIGHT (sframe);
239 for (j = 1; j < height; j++) {
241 for (i = 0; i < width; i++) {
242 SHADE (d, s, i, r, g, b);
249 shader_fade_and_move_left (GstAudioVisualizer * scope,
250 const GstVideoFrame * sframe, GstVideoFrame * dframe)
253 guint r = (scope->priv->shade_amount >> 16) & 0xff;
254 guint g = (scope->priv->shade_amount >> 8) & 0xff;
255 guint b = (scope->priv->shade_amount >> 0) & 0xff;
257 gint ss, ds, width, height;
259 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
260 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
261 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
262 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
264 width = GST_VIDEO_FRAME_WIDTH (sframe);
265 height = GST_VIDEO_FRAME_HEIGHT (sframe);
270 /* move to the left */
271 for (j = 0; j < height; j++) {
272 for (i = 0; i < width; i++) {
273 SHADE (d, s, i, r, g, b);
281 shader_fade_and_move_right (GstAudioVisualizer * scope,
282 const GstVideoFrame * sframe, GstVideoFrame * dframe)
285 guint r = (scope->priv->shade_amount >> 16) & 0xff;
286 guint g = (scope->priv->shade_amount >> 8) & 0xff;
287 guint b = (scope->priv->shade_amount >> 0) & 0xff;
289 gint ss, ds, width, height;
291 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
292 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
293 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
294 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
296 width = GST_VIDEO_FRAME_WIDTH (sframe);
297 height = GST_VIDEO_FRAME_HEIGHT (sframe);
302 /* move to the right */
303 for (j = 0; j < height; j++) {
304 for (i = 0; i < width; i++) {
305 SHADE (d, s, i, r, g, b);
313 shader_fade_and_move_horiz_out (GstAudioVisualizer * scope,
314 const GstVideoFrame * sframe, GstVideoFrame * dframe)
317 guint r = (scope->priv->shade_amount >> 16) & 0xff;
318 guint g = (scope->priv->shade_amount >> 8) & 0xff;
319 guint b = (scope->priv->shade_amount >> 0) & 0xff;
321 gint ss, ds, width, height;
323 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
324 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
325 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
326 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
328 width = GST_VIDEO_FRAME_WIDTH (sframe);
329 height = GST_VIDEO_FRAME_HEIGHT (sframe);
331 /* move upper half up */
332 for (j = 0; j < height / 2; j++) {
334 for (i = 0; i < width; i++) {
335 SHADE (d, s, i, r, g, b);
340 /* rewind one stride */
343 /* move lower half down */
344 for (j = 0; j < height / 2; j++) {
346 for (i = 0; i < width; i++) {
347 SHADE (d, s, i, r, g, b);
354 shader_fade_and_move_horiz_in (GstAudioVisualizer * scope,
355 const GstVideoFrame * sframe, GstVideoFrame * dframe)
358 guint r = (scope->priv->shade_amount >> 16) & 0xff;
359 guint g = (scope->priv->shade_amount >> 8) & 0xff;
360 guint b = (scope->priv->shade_amount >> 0) & 0xff;
362 gint ss, ds, width, height;
364 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
365 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
366 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
367 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
369 width = GST_VIDEO_FRAME_WIDTH (sframe);
370 height = GST_VIDEO_FRAME_HEIGHT (sframe);
372 /* move upper half down */
373 for (j = 0; j < height / 2; j++) {
375 for (i = 0; i < width; i++) {
376 SHADE (d, s, i, r, g, b);
380 /* move lower half up */
381 for (j = 0; j < height / 2; j++) {
383 for (i = 0; i < width; i++) {
384 SHADE (d, s, i, r, g, b);
391 shader_fade_and_move_vert_out (GstAudioVisualizer * scope,
392 const GstVideoFrame * sframe, GstVideoFrame * dframe)
395 guint r = (scope->priv->shade_amount >> 16) & 0xff;
396 guint g = (scope->priv->shade_amount >> 8) & 0xff;
397 guint b = (scope->priv->shade_amount >> 0) & 0xff;
398 guint8 *s, *s1, *d, *d1;
399 gint ss, ds, width, height;
401 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
402 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
403 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
404 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
406 width = GST_VIDEO_FRAME_WIDTH (sframe);
407 height = GST_VIDEO_FRAME_HEIGHT (sframe);
409 for (j = 0; j < height; j++) {
410 /* move left half to the left */
412 for (i = 0; i < width / 2; i++) {
413 SHADE (d, s1, i, r, g, b);
415 /* move right half to the right */
417 for (; i < width - 1; i++) {
418 SHADE (d1, s, i, r, g, b);
426 shader_fade_and_move_vert_in (GstAudioVisualizer * scope,
427 const GstVideoFrame * sframe, GstVideoFrame * dframe)
430 guint r = (scope->priv->shade_amount >> 16) & 0xff;
431 guint g = (scope->priv->shade_amount >> 8) & 0xff;
432 guint b = (scope->priv->shade_amount >> 0) & 0xff;
433 guint8 *s, *s1, *d, *d1;
434 gint ss, ds, width, height;
436 s = GST_VIDEO_FRAME_PLANE_DATA (sframe, 0);
437 ss = GST_VIDEO_FRAME_PLANE_STRIDE (sframe, 0);
438 d = GST_VIDEO_FRAME_PLANE_DATA (dframe, 0);
439 ds = GST_VIDEO_FRAME_PLANE_STRIDE (dframe, 0);
441 width = GST_VIDEO_FRAME_WIDTH (sframe);
442 height = GST_VIDEO_FRAME_HEIGHT (sframe);
444 for (j = 0; j < height; j++) {
445 /* move left half to the right */
447 for (i = 0; i < width / 2; i++) {
448 SHADE (d1, s, i, r, g, b);
450 /* move right half to the left */
452 for (; i < width - 1; i++) {
453 SHADE (d, s1, i, r, g, b);
461 gst_audio_visualizer_change_shader (GstAudioVisualizer * scope)
463 switch (scope->priv->shader_type) {
464 case GST_AUDIO_VISUALIZER_SHADER_NONE:
465 scope->priv->shader = NULL;
467 case GST_AUDIO_VISUALIZER_SHADER_FADE:
468 scope->priv->shader = shader_fade;
470 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_UP:
471 scope->priv->shader = shader_fade_and_move_up;
473 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_DOWN:
474 scope->priv->shader = shader_fade_and_move_down;
476 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_LEFT:
477 scope->priv->shader = shader_fade_and_move_left;
479 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_RIGHT:
480 scope->priv->shader = shader_fade_and_move_right;
482 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_OUT:
483 scope->priv->shader = shader_fade_and_move_horiz_out;
485 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_HORIZ_IN:
486 scope->priv->shader = shader_fade_and_move_horiz_in;
488 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_OUT:
489 scope->priv->shader = shader_fade_and_move_vert_out;
491 case GST_AUDIO_VISUALIZER_SHADER_FADE_AND_MOVE_VERT_IN:
492 scope->priv->shader = shader_fade_and_move_vert_in;
495 GST_ERROR ("invalid shader function");
496 scope->priv->shader = NULL;
504 gst_audio_visualizer_get_type (void)
506 static gsize audio_visualizer_type = 0;
508 if (g_once_init_enter (&audio_visualizer_type)) {
509 static const GTypeInfo audio_visualizer_info = {
510 sizeof (GstAudioVisualizerClass),
513 (GClassInitFunc) gst_audio_visualizer_class_init,
516 sizeof (GstAudioVisualizer),
518 (GInstanceInitFunc) gst_audio_visualizer_init,
522 /* TODO: rename when exporting it as a library */
523 _type = g_type_register_static (GST_TYPE_ELEMENT,
524 "GstAudioVisualizer", &audio_visualizer_info, G_TYPE_FLAG_ABSTRACT);
527 g_type_add_instance_private (_type, sizeof (GstAudioVisualizerPrivate));
529 g_once_init_leave (&audio_visualizer_type, _type);
531 return (GType) audio_visualizer_type;
534 static inline GstAudioVisualizerPrivate *
535 gst_audio_visualizer_get_instance_private (GstAudioVisualizer * self)
537 return (G_STRUCT_MEMBER_P (self, private_offset));
541 gst_audio_visualizer_class_init (GstAudioVisualizerClass * klass)
543 GObjectClass *gobject_class = (GObjectClass *) klass;
544 GstElementClass *element_class = (GstElementClass *) klass;
546 if (private_offset != 0)
547 g_type_class_adjust_private_offset (klass, &private_offset);
549 parent_class = g_type_class_peek_parent (klass);
551 GST_DEBUG_CATEGORY_INIT (audio_visualizer_debug,
552 "baseaudiovisualizer-libvisual", 0,
553 "scope audio visualisation base class");
555 gobject_class->set_property = gst_audio_visualizer_set_property;
556 gobject_class->get_property = gst_audio_visualizer_get_property;
557 gobject_class->dispose = gst_audio_visualizer_dispose;
559 element_class->change_state =
560 GST_DEBUG_FUNCPTR (gst_audio_visualizer_change_state);
562 klass->decide_allocation = GST_DEBUG_FUNCPTR (default_decide_allocation);
564 g_object_class_install_property (gobject_class, PROP_SHADER,
565 g_param_spec_enum ("shader", "shader type",
566 "Shader function to apply on each frame",
567 GST_TYPE_AUDIO_VISUALIZER_SHADER, DEFAULT_SHADER,
568 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
569 g_object_class_install_property (gobject_class, PROP_SHADE_AMOUNT,
570 g_param_spec_uint ("shade-amount", "shade amount",
571 "Shading color to use (big-endian ARGB)", 0, G_MAXUINT32,
572 DEFAULT_SHADE_AMOUNT,
573 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
577 gst_audio_visualizer_init (GstAudioVisualizer * scope,
578 GstAudioVisualizerClass * g_class)
580 GstPadTemplate *pad_template;
582 scope->priv = gst_audio_visualizer_get_instance_private (scope);
584 /* create the sink and src pads */
586 gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "sink");
587 g_return_if_fail (pad_template != NULL);
588 scope->priv->sinkpad = gst_pad_new_from_template (pad_template, "sink");
589 gst_pad_set_chain_function (scope->priv->sinkpad,
590 GST_DEBUG_FUNCPTR (gst_audio_visualizer_chain));
591 gst_pad_set_event_function (scope->priv->sinkpad,
592 GST_DEBUG_FUNCPTR (gst_audio_visualizer_sink_event));
593 gst_element_add_pad (GST_ELEMENT (scope), scope->priv->sinkpad);
596 gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "src");
597 g_return_if_fail (pad_template != NULL);
598 scope->priv->srcpad = gst_pad_new_from_template (pad_template, "src");
599 gst_pad_set_event_function (scope->priv->srcpad,
600 GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_event));
601 gst_pad_set_query_function (scope->priv->srcpad,
602 GST_DEBUG_FUNCPTR (gst_audio_visualizer_src_query));
603 gst_element_add_pad (GST_ELEMENT (scope), scope->priv->srcpad);
605 scope->priv->adapter = gst_adapter_new ();
606 scope->priv->inbuf = gst_buffer_new ();
609 scope->priv->shader_type = DEFAULT_SHADER;
610 gst_audio_visualizer_change_shader (scope);
611 scope->priv->shade_amount = DEFAULT_SHADE_AMOUNT;
613 /* reset the initial video state */
614 gst_video_info_init (&scope->vinfo);
615 scope->priv->frame_duration = GST_CLOCK_TIME_NONE;
617 /* reset the initial state */
618 gst_audio_info_init (&scope->ainfo);
619 gst_video_info_init (&scope->vinfo);
621 g_mutex_init (&scope->priv->config_lock);
625 gst_audio_visualizer_set_property (GObject * object, guint prop_id,
626 const GValue * value, GParamSpec * pspec)
628 GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
632 scope->priv->shader_type = g_value_get_enum (value);
633 gst_audio_visualizer_change_shader (scope);
635 case PROP_SHADE_AMOUNT:
636 scope->priv->shade_amount = g_value_get_uint (value);
639 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
645 gst_audio_visualizer_get_property (GObject * object, guint prop_id,
646 GValue * value, GParamSpec * pspec)
648 GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
652 g_value_set_enum (value, scope->priv->shader_type);
654 case PROP_SHADE_AMOUNT:
655 g_value_set_uint (value, scope->priv->shade_amount);
658 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
664 gst_audio_visualizer_dispose (GObject * object)
666 GstAudioVisualizer *scope = GST_AUDIO_VISUALIZER (object);
668 if (scope->priv->adapter) {
669 g_object_unref (scope->priv->adapter);
670 scope->priv->adapter = NULL;
672 if (scope->priv->inbuf) {
673 gst_buffer_unref (scope->priv->inbuf);
674 scope->priv->inbuf = NULL;
676 if (scope->priv->tempbuf) {
677 gst_video_frame_unmap (&scope->priv->tempframe);
678 gst_buffer_unref (scope->priv->tempbuf);
679 scope->priv->tempbuf = NULL;
681 if (scope->priv->config_lock.p) {
682 g_mutex_clear (&scope->priv->config_lock);
683 scope->priv->config_lock.p = NULL;
685 G_OBJECT_CLASS (parent_class)->dispose (object);
689 gst_audio_visualizer_reset (GstAudioVisualizer * scope)
691 gst_adapter_clear (scope->priv->adapter);
692 gst_segment_init (&scope->priv->segment, GST_FORMAT_UNDEFINED);
694 GST_OBJECT_LOCK (scope);
695 scope->priv->proportion = 1.0;
696 scope->priv->earliest_time = -1;
697 scope->priv->dropped = 0;
698 scope->priv->processed = 0;
699 GST_OBJECT_UNLOCK (scope);
703 gst_audio_visualizer_sink_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
707 if (!gst_audio_info_from_caps (&info, caps))
712 GST_DEBUG_OBJECT (scope, "audio: channels %d, rate %d",
713 GST_AUDIO_INFO_CHANNELS (&info), GST_AUDIO_INFO_RATE (&info));
715 if (!gst_audio_visualizer_src_negotiate (scope)) {
724 GST_WARNING_OBJECT (scope, "could not parse caps");
729 GST_WARNING_OBJECT (scope, "failed to negotiate");
735 gst_audio_visualizer_src_setcaps (GstAudioVisualizer * scope, GstCaps * caps)
738 GstAudioVisualizerClass *klass;
741 if (!gst_video_info_from_caps (&info, caps))
744 klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));
748 scope->priv->frame_duration = gst_util_uint64_scale_int (GST_SECOND,
749 GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
751 gst_util_uint64_scale_int (GST_AUDIO_INFO_RATE (&scope->ainfo),
752 GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info));
753 scope->req_spf = scope->priv->spf;
755 if (scope->priv->tempbuf) {
756 gst_video_frame_unmap (&scope->priv->tempframe);
757 gst_buffer_unref (scope->priv->tempbuf);
759 scope->priv->tempbuf = gst_buffer_new_wrapped (g_malloc0 (scope->vinfo.size),
761 gst_video_frame_map (&scope->priv->tempframe, &scope->vinfo,
762 scope->priv->tempbuf, GST_MAP_READWRITE);
764 if (klass->setup && !klass->setup (scope))
767 GST_DEBUG_OBJECT (scope, "video: dimension %dx%d, framerate %d/%d",
768 GST_VIDEO_INFO_WIDTH (&info), GST_VIDEO_INFO_HEIGHT (&info),
769 GST_VIDEO_INFO_FPS_N (&info), GST_VIDEO_INFO_FPS_D (&info));
770 GST_DEBUG_OBJECT (scope, "blocks: spf %u, req_spf %u",
771 scope->priv->spf, scope->req_spf);
773 gst_pad_set_caps (scope->priv->srcpad, caps);
775 /* find a pool for the negotiated caps now */
776 res = gst_audio_visualizer_do_bufferpool (scope, caps);
777 gst_caps_unref (caps);
784 gst_caps_unref (caps);
785 GST_DEBUG_OBJECT (scope, "error parsing caps");
791 GST_WARNING_OBJECT (scope, "failed to set up");
797 gst_audio_visualizer_src_negotiate (GstAudioVisualizer * scope)
799 GstCaps *othercaps, *target;
800 GstStructure *structure;
804 templ = gst_pad_get_pad_template_caps (scope->priv->srcpad);
806 GST_DEBUG_OBJECT (scope, "performing negotiation");
808 /* see what the peer can do */
809 othercaps = gst_pad_peer_query_caps (scope->priv->srcpad, NULL);
811 target = gst_caps_intersect (othercaps, templ);
812 gst_caps_unref (othercaps);
813 gst_caps_unref (templ);
815 if (gst_caps_is_empty (target))
818 target = gst_caps_truncate (target);
823 target = gst_caps_make_writable (target);
824 structure = gst_caps_get_structure (target, 0);
825 gst_structure_fixate_field_nearest_int (structure, "width", 320);
826 gst_structure_fixate_field_nearest_int (structure, "height", 200);
827 gst_structure_fixate_field_nearest_fraction (structure, "framerate", 25, 1);
828 if (gst_structure_has_field (structure, "pixel-aspect-ratio"))
829 gst_structure_fixate_field_nearest_fraction (structure,
830 "pixel-aspect-ratio", 1, 1);
832 target = gst_caps_fixate (target);
834 GST_DEBUG_OBJECT (scope, "final caps are %" GST_PTR_FORMAT, target);
836 ret = gst_audio_visualizer_src_setcaps (scope, target);
842 gst_caps_unref (target);
847 /* takes ownership of the pool, allocator and query */
849 gst_audio_visualizer_set_allocation (GstAudioVisualizer * scope,
850 GstBufferPool * pool, GstAllocator * allocator,
851 const GstAllocationParams * params, GstQuery * query)
853 GstAllocator *oldalloc;
854 GstBufferPool *oldpool;
856 GstAudioVisualizerPrivate *priv = scope->priv;
858 GST_OBJECT_LOCK (scope);
859 oldpool = priv->pool;
861 priv->pool_active = FALSE;
863 oldalloc = priv->allocator;
864 priv->allocator = allocator;
866 oldquery = priv->query;
870 priv->params = *params;
872 gst_allocation_params_init (&priv->params);
873 GST_OBJECT_UNLOCK (scope);
876 GST_DEBUG_OBJECT (scope, "deactivating old pool %p", oldpool);
877 gst_buffer_pool_set_active (oldpool, FALSE);
878 gst_object_unref (oldpool);
881 gst_object_unref (oldalloc);
884 gst_query_unref (oldquery);
890 gst_audio_visualizer_do_bufferpool (GstAudioVisualizer * scope,
894 gboolean result = TRUE;
895 GstBufferPool *pool = NULL;
896 GstAudioVisualizerClass *klass;
897 GstAllocator *allocator;
898 GstAllocationParams params;
900 /* not passthrough, we need to allocate */
901 /* find a pool for the negotiated caps now */
902 GST_DEBUG_OBJECT (scope, "doing allocation query");
903 query = gst_query_new_allocation (outcaps, TRUE);
905 if (!gst_pad_peer_query (scope->priv->srcpad, query)) {
906 /* not a problem, we use the query defaults */
907 GST_DEBUG_OBJECT (scope, "allocation query failed");
910 klass = GST_AUDIO_VISUALIZER_GET_CLASS (scope);
912 GST_DEBUG_OBJECT (scope, "calling decide_allocation");
913 g_assert (klass->decide_allocation != NULL);
914 result = klass->decide_allocation (scope, query);
916 GST_DEBUG_OBJECT (scope, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result,
920 goto no_decide_allocation;
922 /* we got configuration from our peer or the decide_allocation method,
924 if (gst_query_get_n_allocation_params (query) > 0) {
925 gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms);
928 gst_allocation_params_init (¶ms);
931 if (gst_query_get_n_allocation_pools (query) > 0)
932 gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);
936 gst_audio_visualizer_set_allocation (scope, pool, allocator, ¶ms,
942 no_decide_allocation:
944 GST_WARNING_OBJECT (scope, "Subclass failed to decide allocation");
945 gst_query_unref (query);
952 default_decide_allocation (GstAudioVisualizer * scope, GstQuery * query)
956 guint size, min, max;
957 GstAllocator *allocator;
958 GstAllocationParams params;
959 GstStructure *config;
960 gboolean update_allocator;
961 gboolean update_pool;
963 gst_query_parse_allocation (query, &outcaps, NULL);
965 /* we got configuration from our peer or the decide_allocation method,
967 if (gst_query_get_n_allocation_params (query) > 0) {
968 /* try the allocator */
969 gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms);
970 update_allocator = TRUE;
973 gst_allocation_params_init (¶ms);
974 update_allocator = FALSE;
977 if (gst_query_get_n_allocation_pools (query) > 0) {
978 gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
982 size = GST_VIDEO_INFO_SIZE (&scope->vinfo);
988 /* we did not get a pool, make one ourselves then */
989 pool = gst_video_buffer_pool_new ();
992 config = gst_buffer_pool_get_config (pool);
993 gst_buffer_pool_config_set_params (config, outcaps, size, min, max);
994 gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);
995 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
996 gst_buffer_pool_set_config (pool, config);
998 if (update_allocator)
999 gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms);
1001 gst_query_add_allocation_param (query, allocator, ¶ms);
1004 gst_object_unref (allocator);
1007 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
1009 gst_query_add_allocation_pool (query, pool, size, min, max);
1012 gst_object_unref (pool);
1017 static GstFlowReturn
1018 default_prepare_output_buffer (GstAudioVisualizer * scope, GstBuffer ** outbuf)
1020 GstAudioVisualizerPrivate *priv;
1024 g_assert (priv->pool != NULL);
1026 /* we can't reuse the input buffer */
1027 if (!priv->pool_active) {
1028 GST_DEBUG_OBJECT (scope, "setting pool %p active", priv->pool);
1029 if (!gst_buffer_pool_set_active (priv->pool, TRUE))
1030 goto activate_failed;
1031 priv->pool_active = TRUE;
1033 GST_DEBUG_OBJECT (scope, "using pool alloc");
1035 return gst_buffer_pool_acquire_buffer (priv->pool, outbuf, NULL);
1040 GST_ELEMENT_ERROR (scope, RESOURCE, SETTINGS,
1041 ("failed to activate bufferpool"), ("failed to activate bufferpool"));
1042 return GST_FLOW_ERROR;
1046 static GstFlowReturn
1047 gst_audio_visualizer_chain (GstPad * pad, GstObject * parent,
1050 GstFlowReturn ret = GST_FLOW_OK;
1051 GstAudioVisualizer *scope;
1052 GstAudioVisualizerClass *klass;
1059 scope = GST_AUDIO_VISUALIZER (parent);
1060 klass = GST_AUDIO_VISUALIZER_CLASS (G_OBJECT_GET_CLASS (scope));
1062 GST_LOG_OBJECT (scope, "chainfunc called");
1064 /* resync on DISCONT */
1065 if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT)) {
1066 gst_adapter_clear (scope->priv->adapter);
1069 /* Make sure have an output format */
1070 if (gst_pad_check_reconfigure (scope->priv->srcpad)) {
1071 if (!gst_audio_visualizer_src_negotiate (scope)) {
1072 gst_pad_mark_reconfigure (scope->priv->srcpad);
1073 goto not_negotiated;
1077 rate = GST_AUDIO_INFO_RATE (&scope->ainfo);
1078 bpf = GST_AUDIO_INFO_BPF (&scope->ainfo);
1081 ret = GST_FLOW_NOT_NEGOTIATED;
1085 gst_adapter_push (scope->priv->adapter, buffer);
1087 g_mutex_lock (&scope->priv->config_lock);
1089 /* this is what we want */
1090 sbpf = scope->req_spf * bpf;
1092 inbuf = scope->priv->inbuf;
1093 /* FIXME: the timestamp in the adapter would be different */
1094 gst_buffer_copy_into (inbuf, buffer, GST_BUFFER_COPY_METADATA, 0, -1);
1096 /* this is what we have */
1097 avail = gst_adapter_available (scope->priv->adapter);
1098 GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
1099 while (avail >= sbpf) {
1101 GstVideoFrame outframe;
1103 /* get timestamp of the current adapter content */
1104 ts = gst_adapter_prev_pts (scope->priv->adapter, &dist);
1105 if (GST_CLOCK_TIME_IS_VALID (ts)) {
1106 /* convert bytes to time */
1107 ts += gst_util_uint64_scale_int (dist, GST_SECOND, rate * bpf);
1110 /* check for QoS, don't compute buffers that are known to be late */
1111 if (GST_CLOCK_TIME_IS_VALID (ts)) {
1112 GstClockTime earliest_time;
1117 gst_segment_to_running_time (&scope->priv->segment,
1118 GST_FORMAT_TIME, ts) + scope->priv->frame_duration;
1120 GST_OBJECT_LOCK (scope);
1121 earliest_time = scope->priv->earliest_time;
1122 proportion = scope->priv->proportion;
1123 GST_OBJECT_UNLOCK (scope);
1125 if (GST_CLOCK_TIME_IS_VALID (earliest_time) && qostime <= earliest_time) {
1126 GstClockTime stream_time, jitter;
1127 GstMessage *qos_msg;
1129 GST_DEBUG_OBJECT (scope,
1130 "QoS: skip ts: %" GST_TIME_FORMAT ", earliest: %" GST_TIME_FORMAT,
1131 GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time));
1133 ++scope->priv->dropped;
1134 stream_time = gst_segment_to_stream_time (&scope->priv->segment,
1135 GST_FORMAT_TIME, ts);
1136 jitter = GST_CLOCK_DIFF (qostime, earliest_time);
1137 qos_msg = gst_message_new_qos (GST_OBJECT (scope), FALSE, qostime,
1138 stream_time, ts, GST_BUFFER_DURATION (buffer));
1139 gst_message_set_qos_values (qos_msg, jitter, proportion, 1000000);
1140 gst_message_set_qos_stats (qos_msg, GST_FORMAT_BUFFERS,
1141 scope->priv->processed, scope->priv->dropped);
1142 gst_element_post_message (GST_ELEMENT (scope), qos_msg);
1148 ++scope->priv->processed;
1150 g_mutex_unlock (&scope->priv->config_lock);
1151 ret = default_prepare_output_buffer (scope, &outbuf);
1152 g_mutex_lock (&scope->priv->config_lock);
1153 /* recheck as the value could have changed */
1154 sbpf = scope->req_spf * bpf;
1156 /* no buffer allocated, we don't care why. */
1157 if (ret != GST_FLOW_OK)
1160 /* sync controlled properties */
1161 if (GST_CLOCK_TIME_IS_VALID (ts))
1162 gst_object_sync_values (GST_OBJECT (scope), ts);
1164 GST_BUFFER_PTS (outbuf) = ts;
1165 GST_BUFFER_DURATION (outbuf) = scope->priv->frame_duration;
1167 /* this can fail as the data size we need could have changed */
1168 if (!(adata = (gpointer) gst_adapter_map (scope->priv->adapter, sbpf)))
1171 gst_video_frame_map (&outframe, &scope->vinfo, outbuf,
1172 GST_MAP_READWRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF);
1174 if (scope->priv->shader) {
1175 gst_video_frame_copy (&outframe, &scope->priv->tempframe);
1177 /* gst_video_frame_clear() or is output frame already cleared */
1180 for (i = 0; i < scope->vinfo.finfo->n_planes; i++) {
1181 memset (outframe.data[i], 0, outframe.map[i].size);
1185 gst_buffer_replace_all_memory (inbuf,
1186 gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, adata, sbpf, 0,
1189 /* call class->render() vmethod */
1190 if (klass->render) {
1191 if (!klass->render (scope, inbuf, &outframe)) {
1192 ret = GST_FLOW_ERROR;
1193 gst_video_frame_unmap (&outframe);
1196 /* run various post processing (shading and geometric transformation) */
1197 /* FIXME: SHADER assumes 32bpp */
1198 if (scope->priv->shader &&
1199 GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) {
1200 scope->priv->shader (scope, &outframe, &scope->priv->tempframe);
1204 gst_video_frame_unmap (&outframe);
1206 g_mutex_unlock (&scope->priv->config_lock);
1207 ret = gst_pad_push (scope->priv->srcpad, outbuf);
1209 g_mutex_lock (&scope->priv->config_lock);
1212 /* recheck as the value could have changed */
1213 sbpf = scope->req_spf * bpf;
1214 GST_LOG_OBJECT (scope, "avail: %u, bpf: %u", avail, sbpf);
1215 /* we want to take less or more, depending on spf : req_spf */
1216 if (avail - sbpf >= sbpf) {
1217 gst_adapter_flush (scope->priv->adapter, sbpf);
1218 gst_adapter_unmap (scope->priv->adapter);
1219 } else if (avail >= sbpf) {
1220 /* just flush a bit and stop */
1221 gst_adapter_flush (scope->priv->adapter, (avail - sbpf));
1222 gst_adapter_unmap (scope->priv->adapter);
1225 avail = gst_adapter_available (scope->priv->adapter);
1227 if (ret != GST_FLOW_OK)
1231 g_mutex_unlock (&scope->priv->config_lock);
1239 GST_DEBUG_OBJECT (scope, "Failed to renegotiate");
1240 return GST_FLOW_NOT_NEGOTIATED;
1245 gst_audio_visualizer_src_event (GstPad * pad, GstObject * parent,
1249 GstAudioVisualizer *scope;
1251 scope = GST_AUDIO_VISUALIZER (parent);
1253 switch (GST_EVENT_TYPE (event)) {
1257 GstClockTimeDiff diff;
1258 GstClockTime timestamp;
1260 gst_event_parse_qos (event, NULL, &proportion, &diff, ×tamp);
1262 /* save stuff for the _chain() function */
1263 GST_OBJECT_LOCK (scope);
1264 scope->priv->proportion = proportion;
1266 /* we're late, this is a good estimate for next displayable
1267 * frame (see part-qos.txt) */
1268 scope->priv->earliest_time = timestamp + 2 * diff +
1269 scope->priv->frame_duration;
1271 scope->priv->earliest_time = timestamp + diff;
1272 GST_OBJECT_UNLOCK (scope);
1274 res = gst_pad_push_event (scope->priv->sinkpad, event);
1277 case GST_EVENT_RECONFIGURE:
1279 gst_event_unref (event);
1283 res = gst_pad_event_default (pad, parent, event);
1291 gst_audio_visualizer_sink_event (GstPad * pad, GstObject * parent,
1295 GstAudioVisualizer *scope;
1297 scope = GST_AUDIO_VISUALIZER (parent);
1299 switch (GST_EVENT_TYPE (event)) {
1300 case GST_EVENT_CAPS:
1304 gst_event_parse_caps (event, &caps);
1305 res = gst_audio_visualizer_sink_setcaps (scope, caps);
1306 gst_event_unref (event);
1309 case GST_EVENT_FLUSH_STOP:
1310 gst_audio_visualizer_reset (scope);
1311 res = gst_pad_push_event (scope->priv->srcpad, event);
1313 case GST_EVENT_SEGMENT:
1315 /* the newsegment values are used to clip the input samples
1316 * and to convert the incoming timestamps to running time so
1318 gst_event_copy_segment (event, &scope->priv->segment);
1320 res = gst_pad_push_event (scope->priv->srcpad, event);
1324 res = gst_pad_event_default (pad, parent, event);
1332 gst_audio_visualizer_src_query (GstPad * pad, GstObject * parent,
1335 gboolean res = FALSE;
1336 GstAudioVisualizer *scope;
1338 scope = GST_AUDIO_VISUALIZER (parent);
1340 switch (GST_QUERY_TYPE (query)) {
1341 case GST_QUERY_LATENCY:
1343 /* We need to send the query upstream and add the returned latency to our
1345 GstClockTime min_latency, max_latency;
1347 GstClockTime our_latency;
1349 gint rate = GST_AUDIO_INFO_RATE (&scope->ainfo);
1354 if ((res = gst_pad_peer_query (scope->priv->sinkpad, query))) {
1355 gst_query_parse_latency (query, &us_live, &min_latency, &max_latency);
1357 GST_DEBUG_OBJECT (scope, "Peer latency: min %"
1358 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
1359 GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
1361 /* the max samples we must buffer buffer */
1362 max_samples = MAX (scope->req_spf, scope->priv->spf);
1363 our_latency = gst_util_uint64_scale_int (max_samples, GST_SECOND, rate);
1365 GST_DEBUG_OBJECT (scope, "Our latency: %" GST_TIME_FORMAT,
1366 GST_TIME_ARGS (our_latency));
1368 /* we add some latency but only if we need to buffer more than what
1369 * upstream gives us */
1370 min_latency += our_latency;
1371 if (max_latency != -1)
1372 max_latency += our_latency;
1374 GST_DEBUG_OBJECT (scope, "Calculated total latency : min %"
1375 GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
1376 GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
1378 gst_query_set_latency (query, TRUE, min_latency, max_latency);
1383 res = gst_pad_query_default (pad, parent, query);
1390 static GstStateChangeReturn
1391 gst_audio_visualizer_change_state (GstElement * element,
1392 GstStateChange transition)
1394 GstStateChangeReturn ret;
1395 GstAudioVisualizer *scope;
1397 scope = GST_AUDIO_VISUALIZER (element);
1399 switch (transition) {
1400 case GST_STATE_CHANGE_READY_TO_PAUSED:
1401 gst_audio_visualizer_reset (scope);
1407 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
1409 switch (transition) {
1410 case GST_STATE_CHANGE_PAUSED_TO_READY:
1411 gst_audio_visualizer_set_allocation (scope, NULL, NULL, NULL, NULL);
1413 case GST_STATE_CHANGE_READY_TO_NULL: