2 * WebRTC Audio Processing Elements
4 * Copyright 2016 Collabora Ltd
5 * @author: Nicolas Dufresne <nicolas.dufresne@collabora.com>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 * SECTION:element-webrtcechoprobe
26 * This echo probe is to be used with the webrtcdsp element. See #gst-plugins-bad-plugins-webrtcdsp
27 * documentation for more details.
34 #include "gstwebrtcechoprobe.h"
36 #include <webrtc/modules/interface/module_common_types.h>
37 #include <gst/audio/audio.h>
39 GST_DEBUG_CATEGORY_EXTERN (webrtc_dsp_debug);
40 #define GST_CAT_DEFAULT (webrtc_dsp_debug)
42 #define MAX_ADAPTER_SIZE (1*1024*1024)
44 static GstStaticPadTemplate gst_webrtc_echo_probe_sink_template =
45 GST_STATIC_PAD_TEMPLATE ("sink",
48 GST_STATIC_CAPS ("audio/x-raw, "
49 "format = (string) " GST_AUDIO_NE (S16) ", "
50 "layout = (string) interleaved, "
51 "rate = (int) { 48000, 32000, 16000, 8000 }, "
52 "channels = (int) [1, MAX];"
54 "format = (string) " GST_AUDIO_NE (F32) ", "
55 "layout = (string) non-interleaved, "
56 "rate = (int) { 48000, 32000, 16000, 8000 }, "
57 "channels = (int) [1, MAX]")
60 static GstStaticPadTemplate gst_webrtc_echo_probe_src_template =
61 GST_STATIC_PAD_TEMPLATE ("src",
64 GST_STATIC_CAPS ("audio/x-raw, "
65 "format = (string) " GST_AUDIO_NE (S16) ", "
66 "layout = (string) interleaved, "
67 "rate = (int) { 48000, 32000, 16000, 8000 }, "
68 "channels = (int) [1, MAX];"
70 "format = (string) " GST_AUDIO_NE (F32) ", "
71 "layout = (string) non-interleaved, "
72 "rate = (int) { 48000, 32000, 16000, 8000 }, "
73 "channels = (int) [1, MAX]")
76 G_LOCK_DEFINE_STATIC (gst_aec_probes);
77 static GList *gst_aec_probes = NULL;
79 G_DEFINE_TYPE (GstWebrtcEchoProbe, gst_webrtc_echo_probe,
80 GST_TYPE_AUDIO_FILTER);
83 gst_webrtc_echo_probe_setup (GstAudioFilter * filter, const GstAudioInfo * info)
85 GstWebrtcEchoProbe *self = GST_WEBRTC_ECHO_PROBE (filter);
87 GST_LOG_OBJECT (self, "setting format to %s with %i Hz and %i channels",
88 info->finfo->description, info->rate, info->channels);
90 GST_WEBRTC_ECHO_PROBE_LOCK (self);
93 self->interleaved = (info->layout == GST_AUDIO_LAYOUT_INTERLEAVED);
95 if (!self->interleaved)
96 gst_planar_audio_adapter_configure (self->padapter, info);
98 /* WebRTC library works with 10ms buffers, compute once this size */
99 self->period_samples = info->rate / 100;
100 self->period_size = self->period_samples * info->bpf;
102 if (self->interleaved &&
103 (webrtc::AudioFrame::kMaxDataSizeSamples * 2) < self->period_size)
106 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
111 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
112 GST_WARNING_OBJECT (self, "webrtcdsp format produce too big period "
113 "(maximum is %" G_GSIZE_FORMAT " samples and we have %u samples), "
114 "reduce the number of channels or the rate.",
115 webrtc::AudioFrame::kMaxDataSizeSamples, self->period_size / 2);
120 gst_webrtc_echo_probe_stop (GstBaseTransform * btrans)
122 GstWebrtcEchoProbe *self = GST_WEBRTC_ECHO_PROBE (btrans);
124 GST_WEBRTC_ECHO_PROBE_LOCK (self);
125 gst_adapter_clear (self->adapter);
126 gst_planar_audio_adapter_clear (self->padapter);
127 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
133 gst_webrtc_echo_probe_src_event (GstBaseTransform * btrans, GstEvent * event)
135 GstBaseTransformClass *klass;
136 GstWebrtcEchoProbe *self = GST_WEBRTC_ECHO_PROBE (btrans);
137 GstClockTime latency;
138 GstClockTime upstream_latency = 0;
141 klass = GST_BASE_TRANSFORM_CLASS (gst_webrtc_echo_probe_parent_class);
143 switch (GST_EVENT_TYPE (event)) {
144 case GST_EVENT_LATENCY:
145 gst_event_parse_latency (event, &latency);
146 query = gst_query_new_latency ();
148 if (gst_pad_query (btrans->srcpad, query)) {
149 gst_query_parse_latency (query, NULL, &upstream_latency, NULL);
151 if (!GST_CLOCK_TIME_IS_VALID (upstream_latency))
152 upstream_latency = 0;
155 GST_WEBRTC_ECHO_PROBE_LOCK (self);
156 self->latency = latency;
157 self->delay = upstream_latency / GST_MSECOND;
158 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
160 GST_DEBUG_OBJECT (self, "We have a latency of %" GST_TIME_FORMAT
161 " and delay of %ims", GST_TIME_ARGS (latency),
162 (gint) (upstream_latency / GST_MSECOND));
168 return klass->src_event (btrans, event);
172 gst_webrtc_echo_probe_transform_ip (GstBaseTransform * btrans,
175 GstWebrtcEchoProbe *self = GST_WEBRTC_ECHO_PROBE (btrans);
176 GstBuffer *newbuf = NULL;
178 GST_WEBRTC_ECHO_PROBE_LOCK (self);
179 newbuf = gst_buffer_copy (buffer);
180 /* Moves the buffer timestamp to be in Running time */
181 GST_BUFFER_PTS (newbuf) = gst_segment_to_running_time (&btrans->segment,
182 GST_FORMAT_TIME, GST_BUFFER_PTS (buffer));
184 if (self->interleaved) {
185 gst_adapter_push (self->adapter, newbuf);
187 if (gst_adapter_available (self->adapter) > MAX_ADAPTER_SIZE)
188 gst_adapter_flush (self->adapter,
189 gst_adapter_available (self->adapter) - MAX_ADAPTER_SIZE);
193 gst_planar_audio_adapter_push (self->padapter, newbuf);
195 gst_planar_audio_adapter_available (self->padapter) * self->info.bpf;
196 if (available > MAX_ADAPTER_SIZE)
197 gst_planar_audio_adapter_flush (self->padapter,
198 (available - MAX_ADAPTER_SIZE) / self->info.bpf);
201 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);
207 gst_webrtc_echo_probe_finalize (GObject * object)
209 GstWebrtcEchoProbe *self = GST_WEBRTC_ECHO_PROBE (object);
211 G_LOCK (gst_aec_probes);
212 gst_aec_probes = g_list_remove (gst_aec_probes, self);
213 G_UNLOCK (gst_aec_probes);
215 gst_object_unref (self->adapter);
216 gst_object_unref (self->padapter);
217 self->adapter = NULL;
218 self->padapter = NULL;
220 G_OBJECT_CLASS (gst_webrtc_echo_probe_parent_class)->finalize (object);
224 gst_webrtc_echo_probe_init (GstWebrtcEchoProbe * self)
226 self->adapter = gst_adapter_new ();
227 self->padapter = gst_planar_audio_adapter_new ();
228 gst_audio_info_init (&self->info);
229 g_mutex_init (&self->lock);
231 self->latency = GST_CLOCK_TIME_NONE;
233 G_LOCK (gst_aec_probes);
234 gst_aec_probes = g_list_prepend (gst_aec_probes, self);
235 G_UNLOCK (gst_aec_probes);
239 gst_webrtc_echo_probe_class_init (GstWebrtcEchoProbeClass * klass)
241 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
242 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
243 GstBaseTransformClass *btrans_class = GST_BASE_TRANSFORM_CLASS (klass);
244 GstAudioFilterClass *audiofilter_class = GST_AUDIO_FILTER_CLASS (klass);
246 gobject_class->finalize = gst_webrtc_echo_probe_finalize;
248 btrans_class->passthrough_on_same_caps = TRUE;
249 btrans_class->src_event = GST_DEBUG_FUNCPTR (gst_webrtc_echo_probe_src_event);
250 btrans_class->transform_ip =
251 GST_DEBUG_FUNCPTR (gst_webrtc_echo_probe_transform_ip);
252 btrans_class->stop = GST_DEBUG_FUNCPTR (gst_webrtc_echo_probe_stop);
254 audiofilter_class->setup = GST_DEBUG_FUNCPTR (gst_webrtc_echo_probe_setup);
256 gst_element_class_add_static_pad_template (element_class,
257 &gst_webrtc_echo_probe_src_template);
258 gst_element_class_add_static_pad_template (element_class,
259 &gst_webrtc_echo_probe_sink_template);
261 gst_element_class_set_static_metadata (element_class,
262 "Acoustic Echo Canceller probe",
264 "Gathers playback buffers for webrtcdsp",
265 "Nicolas Dufresne <nicolas.dufrsesne@collabora.com>");
270 gst_webrtc_acquire_echo_probe (const gchar * name)
272 GstWebrtcEchoProbe *ret = NULL;
275 G_LOCK (gst_aec_probes);
276 for (l = gst_aec_probes; l; l = l->next) {
277 GstWebrtcEchoProbe *probe = GST_WEBRTC_ECHO_PROBE (l->data);
279 GST_WEBRTC_ECHO_PROBE_LOCK (probe);
280 if (!probe->acquired && g_strcmp0 (GST_OBJECT_NAME (probe), name) == 0) {
281 probe->acquired = TRUE;
282 ret = GST_WEBRTC_ECHO_PROBE (gst_object_ref (probe));
283 GST_WEBRTC_ECHO_PROBE_UNLOCK (probe);
286 GST_WEBRTC_ECHO_PROBE_UNLOCK (probe);
288 G_UNLOCK (gst_aec_probes);
294 gst_webrtc_release_echo_probe (GstWebrtcEchoProbe * probe)
296 GST_WEBRTC_ECHO_PROBE_LOCK (probe);
297 probe->acquired = FALSE;
298 GST_WEBRTC_ECHO_PROBE_UNLOCK (probe);
299 gst_object_unref (probe);
303 gst_webrtc_echo_probe_read (GstWebrtcEchoProbe * self, GstClockTime rec_time,
304 gpointer _frame, GstBuffer ** buf)
306 webrtc::AudioFrame * frame = (webrtc::AudioFrame *) _frame;
307 GstClockTimeDiff diff;
308 gsize avail, skip, offset, size;
311 GST_WEBRTC_ECHO_PROBE_LOCK (self);
313 if (!GST_CLOCK_TIME_IS_VALID (self->latency) ||
314 !GST_AUDIO_INFO_IS_VALID (&self->info))
317 if (self->interleaved)
318 avail = gst_adapter_available (self->adapter) / self->info.bpf;
320 avail = gst_planar_audio_adapter_available (self->padapter);
322 /* In delay agnostic mode, just return 10ms of data */
323 if (!GST_CLOCK_TIME_IS_VALID (rec_time)) {
324 if (avail < self->period_samples)
327 size = self->period_samples;
337 GstClockTime play_time;
340 if (self->interleaved) {
341 play_time = gst_adapter_prev_pts (self->adapter, &distance);
342 distance /= self->info.bpf;
344 play_time = gst_planar_audio_adapter_prev_pts (self->padapter, &distance);
347 if (GST_CLOCK_TIME_IS_VALID (play_time)) {
348 play_time += gst_util_uint64_scale_int (distance, GST_SECOND,
350 play_time += self->latency;
352 diff = GST_CLOCK_DIFF (rec_time, play_time) / GST_MSECOND;
354 /* We have no timestamp, assume perfect delay */
359 if (diff > self->delay) {
360 skip = (diff - self->delay) * self->info.rate / 1000;
361 skip = MIN (self->period_samples, skip);
365 offset = (self->delay - diff) * self->info.rate / 1000;
366 offset = MIN (avail, offset);
369 size = MIN (avail - offset, self->period_samples - skip);
372 if (self->interleaved) {
373 skip *= self->info.bpf;
374 offset *= self->info.bpf;
375 size *= self->info.bpf;
377 if (size < self->period_size)
378 memset (frame->data_, 0, self->period_size);
381 gst_adapter_copy (self->adapter, (guint8 *) frame->data_ + skip,
383 gst_adapter_flush (self->adapter, offset + size);
386 GstBuffer *ret, *taken, *tmp;
389 gst_planar_audio_adapter_flush (self->padapter, offset);
391 /* we need to fill silence at the beginning and/or the end of each
392 * channel plane in order to have exactly period_samples in the buffer */
393 if (size < self->period_samples) {
395 gint bps = self->info.finfo->width / 8;
396 gsize padding = self->period_samples - (skip + size);
399 taken = gst_planar_audio_adapter_take_buffer (self->padapter, size,
401 meta = gst_buffer_get_audio_meta (taken);
402 ret = gst_buffer_new ();
404 for (c = 0; c < meta->info.channels; c++) {
405 /* need some silence at the beginning */
407 tmp = gst_buffer_new_allocate (NULL, skip * bps, NULL);
408 gst_buffer_memset (tmp, 0, 0, skip * bps);
409 ret = gst_buffer_append (ret, tmp);
412 tmp = gst_buffer_copy_region (taken, GST_BUFFER_COPY_MEMORY,
413 meta->offsets[c], size * bps);
414 ret = gst_buffer_append (ret, tmp);
416 /* need some silence at the end */
418 tmp = gst_buffer_new_allocate (NULL, padding * bps, NULL);
419 gst_buffer_memset (tmp, 0, 0, padding * bps);
420 ret = gst_buffer_append (ret, tmp);
424 gst_buffer_unref (taken);
425 gst_buffer_add_audio_meta (ret, &self->info, self->period_samples,
428 ret = gst_planar_audio_adapter_take_buffer (self->padapter, size,
432 ret = gst_buffer_new_allocate (NULL, self->period_size, NULL);
433 gst_buffer_memset (ret, 0, 0, self->period_size);
434 gst_buffer_add_audio_meta (ret, &self->info, self->period_samples,
441 frame->num_channels_ = self->info.channels;
442 frame->sample_rate_hz_ = self->info.rate;
443 frame->samples_per_channel_ = self->period_samples;
448 GST_WEBRTC_ECHO_PROBE_UNLOCK (self);