2 * Copyright (C) 1999,2000 Erik Walthinsen <omega@cse.ogi.edu>
3 * 2001 Thomas <thomas@apestaart.org>
4 * 2005,2006 Wim Taymans <wim@fluendo.com>
5 * 2013 Sebastian Dröge <sebastian@centricular.com>
7 * audiomixer.c: AudioMixer element, N in, one out, samples are added
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Library General Public
11 * License as published by the Free Software Foundation; either
12 * version 2 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Library General Public License for more details.
19 * You should have received a copy of the GNU Library General Public
20 * License along with this library; if not, write to the
21 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
22 * Boston, MA 02110-1301, USA.
25 * SECTION:element-audiomixer
28 * The audiomixer allows to mix several streams into one by adding the data.
29 * Mixed data is clamped to the min/max values of the data format.
31 * Unlike the adder element audiomixer properly synchronises all input streams
32 * and also handles live inputs such as capture sources or RTP properly.
34 * The audiomixer element can accept any sort of raw audio data, it will
35 * be converted to the target format if necessary, with the exception
36 * of the sample rate, which has to be identical to either what downstream
37 * expects, or the sample rate of the first configured pad. Use a capsfilter
38 * after the audiomixer element if you want to precisely control the format
39 * that comes out of the audiomixer, which supports changing the format of
40 * its output while playing.
42 * If you want to control the manner in which incoming data gets converted,
43 * see the #GstAudioAggregatorConvertPad:converter-config property, which will let
44 * you for example change the way in which channels may get remapped.
46 * The input pads are from a GstPad subclass and have additional
47 * properties to mute each pad individually and set the volume:
49 * * "mute": Whether to mute the pad or not (#gboolean)
50 * * "volume": The volume of the pad, between 0.0 and 10.0 (#gdouble)
52 * ## Example launch line
54 * gst-launch-1.0 audiotestsrc freq=100 ! audiomixer name=mix ! audioconvert ! alsasink audiotestsrc freq=500 ! mix.
55 * ]| This pipeline produces two sine waves mixed together.
63 #include "gstaudiomixer.h"
64 #include <gst/audio/audio.h>
65 #include <string.h> /* strcmp */
66 #include "gstaudiomixerorc.h"
68 #include "gstaudiointerleave.h"
70 #define GST_CAT_DEFAULT gst_audiomixer_debug
71 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
73 #define DEFAULT_PAD_VOLUME (1.0)
74 #define DEFAULT_PAD_MUTE (FALSE)
76 /* some defines for audio processing */
77 /* the volume factor is a range from 0.0 to (arbitrary) VOLUME_MAX_DOUBLE = 10.0
78 * we map 1.0 to VOLUME_UNITY_INT*
80 #define VOLUME_UNITY_INT8 8 /* internal int for unity 2^(8-5) */
81 #define VOLUME_UNITY_INT8_BIT_SHIFT 3 /* number of bits to shift for unity */
82 #define VOLUME_UNITY_INT16 2048 /* internal int for unity 2^(16-5) */
83 #define VOLUME_UNITY_INT16_BIT_SHIFT 11 /* number of bits to shift for unity */
84 #define VOLUME_UNITY_INT24 524288 /* internal int for unity 2^(24-5) */
85 #define VOLUME_UNITY_INT24_BIT_SHIFT 19 /* number of bits to shift for unity */
86 #define VOLUME_UNITY_INT32 134217728 /* internal int for unity 2^(32-5) */
87 #define VOLUME_UNITY_INT32_BIT_SHIFT 27
96 G_DEFINE_TYPE (GstAudioMixerPad, gst_audiomixer_pad,
97 GST_TYPE_AUDIO_AGGREGATOR_CONVERT_PAD);
100 gst_audiomixer_pad_get_property (GObject * object, guint prop_id,
101 GValue * value, GParamSpec * pspec)
103 GstAudioMixerPad *pad = GST_AUDIO_MIXER_PAD (object);
106 case PROP_PAD_VOLUME:
107 g_value_set_double (value, pad->volume);
110 g_value_set_boolean (value, pad->mute);
113 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
119 gst_audiomixer_pad_set_property (GObject * object, guint prop_id,
120 const GValue * value, GParamSpec * pspec)
122 GstAudioMixerPad *pad = GST_AUDIO_MIXER_PAD (object);
125 case PROP_PAD_VOLUME:
126 GST_OBJECT_LOCK (pad);
127 pad->volume = g_value_get_double (value);
128 pad->volume_i8 = pad->volume * VOLUME_UNITY_INT8;
129 pad->volume_i16 = pad->volume * VOLUME_UNITY_INT16;
130 pad->volume_i32 = pad->volume * VOLUME_UNITY_INT32;
131 GST_OBJECT_UNLOCK (pad);
134 GST_OBJECT_LOCK (pad);
135 pad->mute = g_value_get_boolean (value);
136 GST_OBJECT_UNLOCK (pad);
139 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
145 gst_audiomixer_pad_class_init (GstAudioMixerPadClass * klass)
147 GObjectClass *gobject_class = (GObjectClass *) klass;
149 gobject_class->set_property = gst_audiomixer_pad_set_property;
150 gobject_class->get_property = gst_audiomixer_pad_get_property;
152 g_object_class_install_property (gobject_class, PROP_PAD_VOLUME,
153 g_param_spec_double ("volume", "Volume", "Volume of this pad",
154 0.0, 10.0, DEFAULT_PAD_VOLUME,
155 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
156 g_object_class_install_property (gobject_class, PROP_PAD_MUTE,
157 g_param_spec_boolean ("mute", "Mute", "Mute this pad",
159 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
163 gst_audiomixer_pad_init (GstAudioMixerPad * pad)
165 pad->volume = DEFAULT_PAD_VOLUME;
166 pad->mute = DEFAULT_PAD_MUTE;
174 /* These are the formats we can mix natively */
176 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
178 GST_AUDIO_CAPS_MAKE ("{ S32LE, U32LE, S16LE, U16LE, S8, U8, F32LE, F64LE }") \
179 ", layout = interleaved"
182 GST_AUDIO_CAPS_MAKE ("{ S32BE, U32BE, S16BE, U16BE, S8, U8, F32BE, F64BE }") \
183 ", layout = interleaved"
186 static GstStaticPadTemplate gst_audiomixer_src_template =
187 GST_STATIC_PAD_TEMPLATE ("src",
190 GST_STATIC_CAPS (CAPS)
194 GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (GST_AUDIO_FORMATS_ALL) \
195 ", layout=interleaved")
197 static GstStaticPadTemplate gst_audiomixer_sink_template =
198 GST_STATIC_PAD_TEMPLATE ("sink_%u",
203 static void gst_audiomixer_child_proxy_init (gpointer g_iface,
204 gpointer iface_data);
206 #define gst_audiomixer_parent_class parent_class
207 G_DEFINE_TYPE_WITH_CODE (GstAudioMixer, gst_audiomixer,
208 GST_TYPE_AUDIO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
209 gst_audiomixer_child_proxy_init));
211 static GstPad *gst_audiomixer_request_new_pad (GstElement * element,
212 GstPadTemplate * temp, const gchar * req_name, const GstCaps * caps);
213 static void gst_audiomixer_release_pad (GstElement * element, GstPad * pad);
216 gst_audiomixer_aggregate_one_buffer (GstAudioAggregator * aagg,
217 GstAudioAggregatorPad * aaggpad, GstBuffer * inbuf, guint in_offset,
218 GstBuffer * outbuf, guint out_offset, guint num_samples);
222 gst_audiomixer_class_init (GstAudioMixerClass * klass)
224 GstElementClass *gstelement_class = (GstElementClass *) klass;
225 GstAudioAggregatorClass *aagg_class = (GstAudioAggregatorClass *) klass;
227 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
228 &gst_audiomixer_src_template, GST_TYPE_AUDIO_AGGREGATOR_CONVERT_PAD);
229 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
230 &gst_audiomixer_sink_template, GST_TYPE_AUDIO_MIXER_PAD);
231 gst_element_class_set_static_metadata (gstelement_class, "AudioMixer",
232 "Generic/Audio", "Mixes multiple audio streams",
233 "Sebastian Dröge <sebastian@centricular.com>");
235 gstelement_class->request_new_pad =
236 GST_DEBUG_FUNCPTR (gst_audiomixer_request_new_pad);
237 gstelement_class->release_pad =
238 GST_DEBUG_FUNCPTR (gst_audiomixer_release_pad);
240 aagg_class->aggregate_one_buffer = gst_audiomixer_aggregate_one_buffer;
242 gst_type_mark_as_plugin_api (GST_TYPE_AUDIO_MIXER_PAD, 0);
246 gst_audiomixer_init (GstAudioMixer * audiomixer)
251 gst_audiomixer_request_new_pad (GstElement * element, GstPadTemplate * templ,
252 const gchar * req_name, const GstCaps * caps)
254 GstAudioMixerPad *newpad;
256 newpad = (GstAudioMixerPad *)
257 GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
258 templ, req_name, caps);
261 goto could_not_create;
263 gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad),
264 GST_OBJECT_NAME (newpad));
266 return GST_PAD_CAST (newpad);
270 GST_DEBUG_OBJECT (element, "could not create/add pad");
276 gst_audiomixer_release_pad (GstElement * element, GstPad * pad)
278 GstAudioMixer *audiomixer;
280 audiomixer = GST_AUDIO_MIXER (element);
282 GST_DEBUG_OBJECT (audiomixer, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad));
284 gst_child_proxy_child_removed (GST_CHILD_PROXY (audiomixer), G_OBJECT (pad),
285 GST_OBJECT_NAME (pad));
287 GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
292 gst_audiomixer_aggregate_one_buffer (GstAudioAggregator * aagg,
293 GstAudioAggregatorPad * aaggpad, GstBuffer * inbuf, guint in_offset,
294 GstBuffer * outbuf, guint out_offset, guint num_frames)
296 GstAudioMixerPad *pad = GST_AUDIO_MIXER_PAD (aaggpad);
300 GstAggregator *agg = GST_AGGREGATOR (aagg);
301 GstAudioAggregatorPad *srcpad = GST_AUDIO_AGGREGATOR_PAD (agg->srcpad);
303 GST_OBJECT_LOCK (aagg);
304 GST_OBJECT_LOCK (aaggpad);
306 if (pad->mute || pad->volume < G_MINDOUBLE) {
307 GST_DEBUG_OBJECT (pad, "Skipping muted pad");
308 GST_OBJECT_UNLOCK (aaggpad);
309 GST_OBJECT_UNLOCK (aagg);
313 bpf = GST_AUDIO_INFO_BPF (&srcpad->info);
315 gst_buffer_map (outbuf, &outmap, GST_MAP_READWRITE);
316 gst_buffer_map (inbuf, &inmap, GST_MAP_READ);
317 GST_LOG_OBJECT (pad, "mixing %u bytes at offset %u from offset %u",
318 num_frames * bpf, out_offset * bpf, in_offset * bpf);
320 /* further buffers, need to add them */
321 if (pad->volume == 1.0) {
322 switch (srcpad->info.finfo->format) {
323 case GST_AUDIO_FORMAT_U8:
324 audiomixer_orc_add_u8 ((gpointer) (outmap.data + out_offset * bpf),
325 (gpointer) (inmap.data + in_offset * bpf),
326 num_frames * srcpad->info.channels);
328 case GST_AUDIO_FORMAT_S8:
329 audiomixer_orc_add_s8 ((gpointer) (outmap.data + out_offset * bpf),
330 (gpointer) (inmap.data + in_offset * bpf),
331 num_frames * srcpad->info.channels);
333 case GST_AUDIO_FORMAT_U16:
334 audiomixer_orc_add_u16 ((gpointer) (outmap.data + out_offset * bpf),
335 (gpointer) (inmap.data + in_offset * bpf),
336 num_frames * srcpad->info.channels);
338 case GST_AUDIO_FORMAT_S16:
339 audiomixer_orc_add_s16 ((gpointer) (outmap.data + out_offset * bpf),
340 (gpointer) (inmap.data + in_offset * bpf),
341 num_frames * srcpad->info.channels);
343 case GST_AUDIO_FORMAT_U32:
344 audiomixer_orc_add_u32 ((gpointer) (outmap.data + out_offset * bpf),
345 (gpointer) (inmap.data + in_offset * bpf),
346 num_frames * srcpad->info.channels);
348 case GST_AUDIO_FORMAT_S32:
349 audiomixer_orc_add_s32 ((gpointer) (outmap.data + out_offset * bpf),
350 (gpointer) (inmap.data + in_offset * bpf),
351 num_frames * srcpad->info.channels);
353 case GST_AUDIO_FORMAT_F32:
354 audiomixer_orc_add_f32 ((gpointer) (outmap.data + out_offset * bpf),
355 (gpointer) (inmap.data + in_offset * bpf),
356 num_frames * srcpad->info.channels);
358 case GST_AUDIO_FORMAT_F64:
359 audiomixer_orc_add_f64 ((gpointer) (outmap.data + out_offset * bpf),
360 (gpointer) (inmap.data + in_offset * bpf),
361 num_frames * srcpad->info.channels);
364 g_assert_not_reached ();
368 switch (srcpad->info.finfo->format) {
369 case GST_AUDIO_FORMAT_U8:
370 audiomixer_orc_add_volume_u8 ((gpointer) (outmap.data +
371 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
372 pad->volume_i8, num_frames * srcpad->info.channels);
374 case GST_AUDIO_FORMAT_S8:
375 audiomixer_orc_add_volume_s8 ((gpointer) (outmap.data +
376 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
377 pad->volume_i8, num_frames * srcpad->info.channels);
379 case GST_AUDIO_FORMAT_U16:
380 audiomixer_orc_add_volume_u16 ((gpointer) (outmap.data +
381 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
382 pad->volume_i16, num_frames * srcpad->info.channels);
384 case GST_AUDIO_FORMAT_S16:
385 audiomixer_orc_add_volume_s16 ((gpointer) (outmap.data +
386 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
387 pad->volume_i16, num_frames * srcpad->info.channels);
389 case GST_AUDIO_FORMAT_U32:
390 audiomixer_orc_add_volume_u32 ((gpointer) (outmap.data +
391 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
392 pad->volume_i32, num_frames * srcpad->info.channels);
394 case GST_AUDIO_FORMAT_S32:
395 audiomixer_orc_add_volume_s32 ((gpointer) (outmap.data +
396 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
397 pad->volume_i32, num_frames * srcpad->info.channels);
399 case GST_AUDIO_FORMAT_F32:
400 audiomixer_orc_add_volume_f32 ((gpointer) (outmap.data +
401 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
402 pad->volume, num_frames * srcpad->info.channels);
404 case GST_AUDIO_FORMAT_F64:
405 audiomixer_orc_add_volume_f64 ((gpointer) (outmap.data +
406 out_offset * bpf), (gpointer) (inmap.data + in_offset * bpf),
407 pad->volume, num_frames * srcpad->info.channels);
410 g_assert_not_reached ();
414 gst_buffer_unmap (inbuf, &inmap);
415 gst_buffer_unmap (outbuf, &outmap);
417 GST_OBJECT_UNLOCK (aaggpad);
418 GST_OBJECT_UNLOCK (aagg);
424 /* GstChildProxy implementation */
426 gst_audiomixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
429 GstAudioMixer *audiomixer = GST_AUDIO_MIXER (child_proxy);
432 GST_OBJECT_LOCK (audiomixer);
433 obj = g_list_nth_data (GST_ELEMENT_CAST (audiomixer)->sinkpads, index);
435 gst_object_ref (obj);
436 GST_OBJECT_UNLOCK (audiomixer);
442 gst_audiomixer_child_proxy_get_children_count (GstChildProxy * child_proxy)
445 GstAudioMixer *audiomixer = GST_AUDIO_MIXER (child_proxy);
447 GST_OBJECT_LOCK (audiomixer);
448 count = GST_ELEMENT_CAST (audiomixer)->numsinkpads;
449 GST_OBJECT_UNLOCK (audiomixer);
450 GST_INFO_OBJECT (audiomixer, "Children Count: %d", count);
456 gst_audiomixer_child_proxy_init (gpointer g_iface, gpointer iface_data)
458 GstChildProxyInterface *iface = g_iface;
460 GST_INFO ("initializing child proxy interface");
461 iface->get_child_by_index = gst_audiomixer_child_proxy_get_child_by_index;
462 iface->get_children_count = gst_audiomixer_child_proxy_get_children_count;
465 /* Empty liveadder alias with non-zero latency */
467 typedef GstAudioMixer GstLiveAdder;
468 typedef GstAudioMixerClass GstLiveAdderClass;
470 static GType gst_live_adder_get_type (void);
471 #define GST_TYPE_LIVE_ADDER gst_live_adder_get_type ()
473 G_DEFINE_TYPE (GstLiveAdder, gst_live_adder, GST_TYPE_AUDIO_MIXER);
477 LIVEADDER_PROP_LATENCY = 1
481 gst_live_adder_init (GstLiveAdder * self)
486 gst_live_adder_set_property (GObject * object, guint prop_id,
487 const GValue * value, GParamSpec * pspec)
490 case LIVEADDER_PROP_LATENCY:
492 GParamSpec *parent_spec =
493 g_object_class_find_property (G_OBJECT_CLASS
494 (gst_live_adder_parent_class), "latency");
495 GObjectClass *pspec_class = g_type_class_peek (parent_spec->owner_type);
498 g_value_init (&v, G_TYPE_UINT64);
500 g_value_set_uint64 (&v, g_value_get_uint (value) * GST_MSECOND);
502 G_OBJECT_CLASS (pspec_class)->set_property (object,
503 parent_spec->param_id, &v, parent_spec);
507 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
513 gst_live_adder_get_property (GObject * object, guint prop_id, GValue * value,
517 case LIVEADDER_PROP_LATENCY:
519 GParamSpec *parent_spec =
520 g_object_class_find_property (G_OBJECT_CLASS
521 (gst_live_adder_parent_class), "latency");
522 GObjectClass *pspec_class = g_type_class_peek (parent_spec->owner_type);
525 g_value_init (&v, G_TYPE_UINT64);
527 G_OBJECT_CLASS (pspec_class)->get_property (object,
528 parent_spec->param_id, &v, parent_spec);
530 g_value_set_uint (value, g_value_get_uint64 (&v) / GST_MSECOND);
534 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
541 gst_live_adder_class_init (GstLiveAdderClass * klass)
543 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
545 gobject_class->set_property = gst_live_adder_set_property;
546 gobject_class->get_property = gst_live_adder_get_property;
548 g_object_class_install_property (gobject_class, LIVEADDER_PROP_LATENCY,
549 g_param_spec_uint ("latency", "Buffer latency",
550 "Additional latency in live mode to allow upstream "
551 "to take longer to produce buffers for the current "
552 "position (in milliseconds)", 0, G_MAXUINT,
553 30, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT));
557 plugin_init (GstPlugin * plugin)
559 GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "audiomixer", 0,
560 "audio mixing element");
562 if (!gst_element_register (plugin, "audiomixer", GST_RANK_NONE,
563 GST_TYPE_AUDIO_MIXER))
566 if (!gst_element_register (plugin, "liveadder", GST_RANK_NONE,
567 GST_TYPE_LIVE_ADDER))
570 if (!gst_element_register (plugin, "audiointerleave", GST_RANK_NONE,
571 GST_TYPE_AUDIO_INTERLEAVE))
577 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
580 "Mixes multiple audio streams",
581 plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)