From: Jaechul Lee Date: Mon, 14 Mar 2022 06:29:30 +0000 (+0900) Subject: Support webrtc-audio-processsing echo-cancellation X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=3a68281c2ccfeee47471e869bca3725637775730;p=platform%2Fcore%2Fmultimedia%2Fpulseaudio-modules-tizen.git Support webrtc-audio-processsing echo-cancellation [Version] 15.0.5 [Issue Type] New Feature Change-Id: Ib3227c185779353747af7f930de13535988b467c Signed-off-by: Jaechul Lee --- diff --git a/Makefile.am b/Makefile.am index 768114b..c41e653 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,12 @@ libprocessor_la_LDFLAGS = $(AM_LDFLAGS) $(PA_LDFLAGS) -avoid-version libprocessor_la_LIBADD = $(AM_LIBADD) $(LIBSPEEX_LIBS) libprocessor_la_CFLAGS = $(AM_CFLAGS) $(PA_CFLAGS) $(LIBSPEEX_CFLAGS) +if ENABLE_WEBRTC +libprocessor_la_SOURCES += src/echo-cancel/algo_webrtc.cpp +libprocessor_la_LIBADD += $(WEBRTC_LIBS) +libprocessor_la_CPPFLAGS = $(WEBRTC_CFLAGS) $(PA_CFLAGS) -std=c++17 +endif + module_tizenaudio_echo_cancel_la_SOURCES = src/echo-cancel/module-tizenaudio-echo-cancel.c src/echo-cancel/echo-cancel-def.h module_tizenaudio_echo_cancel_la_LDFLAGS = $(MODULE_LDFLAGS) module_tizenaudio_echo_cancel_la_LIBADD = $(MODULE_LIBADD) libprocessor.la diff --git a/configure.ac b/configure.ac index de19973..170b340 100644 --- a/configure.ac +++ b/configure.ac @@ -409,6 +409,25 @@ AC_ARG_ENABLE(aec, AC_HELP_STRING([--enable-aec], [using aec]), AM_CONDITIONAL(ENABLE_AEC, test "x$ENABLE_AEC" = "xyes") dnl end -------------------------------------------------------------------- +dnl use webrtc ---------------------------------------------------------------- +AC_ARG_ENABLE(webrtc, AC_HELP_STRING([--enable-webrtc], [using webrtc-audio-processing]), +[ + case "${enableval}" in + yes) ENABLE_WEBRTC=yes ;; + no) ENABLE_WEBRTC=no ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-webrtc) ;; + esac + ],[USE_WEBRTC=no]) + +if test "x$ENABLE_WEBRTC" = "xyes"; then +PKG_CHECK_MODULES(WEBRTC, webrtc-audio-processing) +AC_SUBST(WEBRTC_CFLAGS) +AC_SUBST(WEBRTC_LIBS) +fi + +AM_CONDITIONAL(ENABLE_WEBRTC, test "x$ENABLE_WEBRTC" = "xyes") +dnl end -------------------------------------------------------------------- + #### D-Bus support (optional) #### AC_ARG_ENABLE([dbus], diff --git a/packaging/pulseaudio-modules-tizen.spec b/packaging/pulseaudio-modules-tizen.spec index 1a86b8f..11f3e2f 100644 --- a/packaging/pulseaudio-modules-tizen.spec +++ b/packaging/pulseaudio-modules-tizen.spec @@ -2,7 +2,7 @@ Name: pulseaudio-modules-tizen Summary: Pulseaudio modules for Tizen -Version: 15.0.4 +Version: 15.0.5 Release: 0 Group: Multimedia/Audio License: LGPL-2.1+ diff --git a/src/echo-cancel/algo_webrtc.cpp b/src/echo-cancel/algo_webrtc.cpp new file mode 100644 index 0000000..8253820 --- /dev/null +++ b/src/echo-cancel/algo_webrtc.cpp @@ -0,0 +1,286 @@ +/*** + This file is part of PulseAudio. + + Copyright 2021 Jaechul Lee + + PulseAudio is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published + by the Free Software Foundation; either version 2.1 of the License, + or (at your option) any later version. + + PulseAudio is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with PulseAudio; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + USA. +***/ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#define DEFAULT_PROCESS_SIZE_MS 10 + +using namespace webrtc; + +PA_C_DECL_BEGIN +#include +#include +#include +#include +#include +#include +#include +#include + +static void allocate_stream_buffer(struct algo_webrtc *webrtc, size_t nframes); +static void deallocate_stream_buffer(struct algo_webrtc *webrtc); +static void convert_s16_to_float(float *dst, int16_t *src, size_t n); +static void convert_float_to_s16(int16_t *dst, float *src, size_t n); + +void *webrtc_audio_create(size_t nframes, pa_sample_spec *ss); +int32_t webrtc_audio_process(void *priv, int8_t *rec, int8_t *ref, int8_t *out); +int32_t webrtc_audio_destroy(void *priv); +PA_C_DECL_END + +struct algo_webrtc { + AudioProcessing *ap; + Config config; + StreamConfig *sconfig; + + float *rec_fbuf; + float *rec_dbuf[PA_CHANNELS_MAX]; + + float *ref_fbuf; + float *ref_dbuf[PA_CHANNELS_MAX]; + + float *out_fbuf; + float *out_dbuf[PA_CHANNELS_MAX]; + + pa_sample_spec ss; + size_t frames; + + /* Currently, webrtc uses fixed size(10ms) buffer */ + int loops; + size_t fixed_bytes; + size_t fixed_frames; +}; + +void *webrtc_audio_create(size_t nframes, pa_sample_spec *ss) { + struct algo_webrtc *webrtc = NULL; + size_t fixed_bytes, request_bytes; + Config config; + + pa_assert(ss); + + if (ss->channels > 2 || ss->format != PA_SAMPLE_S16LE) { + pa_log_error("Invalid channels (%d) or format (%d)", ss->channels, ss->format); + return NULL; + } + + fixed_bytes = pa_usec_to_bytes(DEFAULT_PROCESS_SIZE_MS * PA_USEC_PER_MSEC, ss); + request_bytes = nframes * pa_frame_size(ss); + + if (fixed_bytes > request_bytes) { + pa_log_error("nframes should be bigger than %dms. nframes(%zu) request_bytes(%zu)", + DEFAULT_PROCESS_SIZE_MS, nframes, request_bytes); + return NULL; + } + + if (request_bytes % fixed_bytes) { + pa_log_error("request_bytes(%zu) should be multiples of fixed_bytes(%zu)", + nframes, request_bytes); + return NULL; + } + + webrtc = pa_xnew0(struct algo_webrtc, 1); + webrtc->ss = *ss; + webrtc->fixed_bytes = fixed_bytes; + webrtc->fixed_frames = fixed_bytes / pa_frame_size(ss); + webrtc->loops = request_bytes / fixed_bytes; + + config.Set(new ExperimentalNs(false)); + config.Set(new Intelligibility(false)); + config.Set(new ExperimentalAgc(false)); + + webrtc->ap = AudioProcessing::Create(config); + if (!webrtc->ap) { + pa_log_error("Failed to create audio processing"); + goto fail; + } + + webrtc->ap->echo_cancellation()->Enable(true); + webrtc->ap->noise_suppression()->Enable(true); + webrtc->ap->noise_suppression()->set_level(static_cast(1)); + + webrtc->ap->gain_control()->set_mode(GainControl::kAdaptiveDigital); + //webrtc->ap->gain_control()->set_target_level_dbfs(30); + //webrtc->ap->gain_control()->set_stream_analog_level(30); + webrtc->ap->echo_cancellation()->set_suppression_level(static_cast(1)); + + webrtc->sconfig = new StreamConfig(ss->rate, ss->channels, false); + if (!webrtc->sconfig) { + pa_log_error("Failed to create stream config"); + goto fail; + } + + webrtc->sconfig->set_sample_rate_hz(ss->rate); + webrtc->sconfig->set_num_channels(ss->channels); + + /* webrtc supports 10ms by default */ + webrtc->frames = webrtc->sconfig->num_frames(); + if (webrtc->frames != webrtc->fixed_frames) { + pa_log_error("Failed to set frames. frames(%zu), fixed_frames(%zu)", + webrtc->frames, webrtc->fixed_frames); + goto fail; + } + + allocate_stream_buffer(webrtc, webrtc->fixed_frames); + + pa_log_info("webrtc processes request block(%llumsec) block(%llumsec) n(%d)\n", + pa_bytes_to_usec(request_bytes, ss) / PA_USEC_PER_MSEC, + pa_bytes_to_usec(fixed_bytes, ss) / PA_USEC_PER_MSEC, + webrtc->loops); + + return webrtc; + +fail: + if (webrtc->ap) + delete webrtc->ap; + if (webrtc->sconfig) + delete webrtc->sconfig; + + pa_xfree(webrtc); + + return NULL; +} + +int32_t webrtc_audio_process(void *priv, int8_t *rec, int8_t *ref, int8_t *out) { + struct algo_webrtc *webrtc = (struct algo_webrtc *)priv; + pa_sample_spec ss; + size_t frames; + + pa_assert(webrtc); + pa_assert(rec); + pa_assert(ref); + pa_assert(out); + + ss.format = PA_SAMPLE_FLOAT32LE; + ss.rate = webrtc->ss.rate; + ss.channels = webrtc->ss.channels; + + frames = webrtc->fixed_frames; + + for (int i = 0; i < webrtc->loops; i++) { + int ret; + + convert_s16_to_float(webrtc->ref_fbuf, (int16_t *)ref, frames * ss.channels); + pa_deinterleave(webrtc->ref_fbuf, (void **)webrtc->ref_dbuf, ss.channels, + pa_sample_size(&ss), frames); + + /* reference */ + ret = webrtc->ap->ProcessReverseStream(webrtc->ref_dbuf, + *webrtc->sconfig, + *webrtc->sconfig, + webrtc->ref_dbuf); + if (ret != AudioProcessing::kNoError) { + pa_log_error("Failed to process reverse stream"); + return -1; + } + + webrtc->ap->set_stream_delay_ms(0); + + /* capture */ + convert_s16_to_float(webrtc->rec_fbuf, (int16_t *)rec, frames * ss.channels); + pa_deinterleave(webrtc->rec_fbuf, (void **)webrtc->rec_dbuf, ss.channels, + pa_sample_size(&ss), frames); + + ret = webrtc->ap->ProcessStream(webrtc->rec_dbuf, + *webrtc->sconfig, + *webrtc->sconfig, + webrtc->out_dbuf); + if (ret != AudioProcessing::kNoError) { + pa_log_error("Failed to process stream"); + return -1; + } + + pa_interleave((const void **)webrtc->out_dbuf, ss.channels, webrtc->out_fbuf, + pa_sample_size(&ss), frames); + convert_float_to_s16((int16_t *)out, webrtc->out_fbuf, frames * ss.channels); + + rec += webrtc->fixed_bytes; + ref += webrtc->fixed_bytes; + out += webrtc->fixed_bytes; + } + + return 0; +} + +int32_t webrtc_audio_destroy(void *priv) { + struct algo_webrtc *webrtc = (struct algo_webrtc *)priv; + + pa_assert(webrtc); + + delete webrtc->sconfig; + delete webrtc->ap; + + deallocate_stream_buffer(webrtc); + pa_xfree(webrtc); + + return 0; +} + +static void deallocate_stream_buffer(struct algo_webrtc *webrtc) { + pa_assert(webrtc); + + for (int i = 0; i < webrtc->ss.channels; i++) { + pa_xfree(webrtc->rec_dbuf[i]); + pa_xfree(webrtc->ref_dbuf[i]); + pa_xfree(webrtc->out_dbuf[i]); + } + + pa_xfree(webrtc->rec_fbuf); + pa_xfree(webrtc->ref_fbuf); + pa_xfree(webrtc->out_fbuf); +} + +static void allocate_stream_buffer(struct algo_webrtc *webrtc, size_t nframes) { + int channels; + + pa_assert(webrtc); + + channels = webrtc->ss.channels; + + webrtc->rec_fbuf = pa_xnew(float, nframes * channels); + webrtc->ref_fbuf = pa_xnew(float, nframes * channels); + webrtc->out_fbuf = pa_xnew(float, nframes * channels); + + for (int i = 0; i < channels; i++) { + webrtc->rec_dbuf[i] = pa_xnew(float, nframes); + webrtc->ref_dbuf[i] = pa_xnew(float, nframes); + webrtc->out_dbuf[i] = pa_xnew(float, nframes); + } +} + +static void convert_s16_to_float(float *dst, int16_t *src, size_t n) { + pa_assert(dst); + pa_assert(src); + + ((pa_convert_func_t)pa_get_convert_to_float32ne_function(PA_SAMPLE_S16LE))(n, src, dst); +} + +static void convert_float_to_s16(int16_t *dst, float *src, size_t n) { + pa_assert(dst); + pa_assert(src); + + ((pa_convert_func_t)pa_get_convert_to_s16ne_function(PA_SAMPLE_FLOAT32LE))(n, src, dst); +} + diff --git a/src/echo-cancel/module-tizenaudio-echo-cancel.c b/src/echo-cancel/module-tizenaudio-echo-cancel.c index 33482eb..a80eb9f 100644 --- a/src/echo-cancel/module-tizenaudio-echo-cancel.c +++ b/src/echo-cancel/module-tizenaudio-echo-cancel.c @@ -125,6 +125,10 @@ static pa_processor_algo_t pa_processor_get_algo(pa_source_output *o) { return PA_PROCESSOR_ADRIAN; else if (pa_streq(algo, "speex")) return PA_PROCESSOR_SPEEX; + else if (pa_streq(algo, "webrtc")) + return PA_PROCESSOR_WEBRTC; + else if (pa_streq(algo, "auto")) + return PA_PROCESSOR_WEBRTC; else { pa_log_warn("invalid algo(%s), Use default processor(speex)", algo); return PA_PROCESSOR_SPEEX; diff --git a/src/echo-cancel/processor.c b/src/echo-cancel/processor.c index e941c0e..3681347 100644 --- a/src/echo-cancel/processor.c +++ b/src/echo-cancel/processor.c @@ -51,6 +51,10 @@ extern void *speex_create(size_t nframes, pa_sample_spec *ss); extern int32_t speex_process(void *priv, int8_t *rec, int8_t *ref, int8_t *out); extern int32_t speex_destroy(void *priv); +extern void *webrtc_audio_create(size_t nframes, pa_sample_spec *ss); +extern int32_t webrtc_audio_process(void *priv, int8_t *rec, int8_t *ref, int8_t *out); +extern int32_t webrtc_audio_destroy(void *priv); + pa_processor *pa_processor_new(size_t nframes, pa_sample_spec *ss, pa_processor_algo_t backend, pa_process_flags_t flags) { pa_processor *processor = NULL; @@ -77,6 +81,11 @@ pa_processor *pa_processor_new(size_t nframes, pa_sample_spec *ss, pa_processor_ processor->intf->process = adrian_process; processor->intf->destroy = adrian_destroy; break; + case PA_PROCESSOR_WEBRTC: + processor->intf->create = webrtc_audio_create; + processor->intf->process = webrtc_audio_process; + processor->intf->destroy = webrtc_audio_destroy; + break; default: pa_log_error("Invalid backend(%d)", backend); goto fail; diff --git a/src/echo-cancel/processor.h b/src/echo-cancel/processor.h index 98482ab..fa642ce 100644 --- a/src/echo-cancel/processor.h +++ b/src/echo-cancel/processor.h @@ -38,6 +38,7 @@ typedef enum { typedef enum { PA_PROCESSOR_SPEEX, PA_PROCESSOR_ADRIAN, + PA_PROCESSOR_WEBRTC, } pa_processor_algo_t; typedef struct pa_processor_algo pa_processor_algo;