1 /***
2 This file is part of PulseAudio.
3
4 Copyright 2011 Collabora Ltd.
5 2015 Aldebaran SoftBank Group
6 2020 Arun Raghavan <arun@asymptotic.io>
7 2020 Eero Nurkkala <eero.nurkkala@offcode.fi>
8
9 PulseAudio is free software; you can redistribute it and/or modify
10 it under the terms of the GNU Lesser General Public License as published
11 by the Free Software Foundation; either version 2.1 of the License,
12 or (at your option) any later version.
13
14 PulseAudio is distributed in the hope that it will be useful, but
15 WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 General Public License for more details.
18
19 You should have received a copy of the GNU Lesser General Public License
20 along with PulseAudio; if not, see <http://www.gnu.org/licenses/>.
21 ***/
22
23 #ifdef HAVE_CONFIG_H
24 #include <config.h>
25 #endif
26
27 #include <pulse/cdecl.h>
28
29 PA_C_DECL_BEGIN
30 #include <pulsecore/core-util.h>
31 #include <pulsecore/modargs.h>
32
33 #include <pulse/timeval.h>
34 #include "echo-cancel.h"
35 PA_C_DECL_END
36
37 #define WEBRTC_APM_DEBUG_DUMP 0
38
39 #include <modules/audio_processing/include/audio_processing.h>
40
41 #define BLOCK_SIZE_US 10000
42
43 #define DEFAULT_HIGH_PASS_FILTER true
44 #define DEFAULT_NOISE_SUPPRESSION true
45 #define DEFAULT_TRANSIENT_NOISE_SUPPRESSION true
46 #define DEFAULT_ANALOG_GAIN_CONTROL true
47 #define DEFAULT_DIGITAL_GAIN_CONTROL false
48 #define DEFAULT_MOBILE false
49 #define DEFAULT_COMFORT_NOISE true
50 #define DEFAULT_DRIFT_COMPENSATION false
51 #define DEFAULT_VAD false
52 #define DEFAULT_AGC_START_VOLUME 85
53 #define DEFAULT_POSTAMP_ENABLE false
54 #define DEFAULT_POSTAMP_GAIN_DB 0
55 #define DEFAULT_PREAMP_ENABLE false
56 #define DEFAULT_PREAMP_GAIN_DB 0
57
58 #define WEBRTC_AGC_MAX_VOLUME 255
59 #define WEBRTC_POSTAMP_GAIN_MAX_DB 90
60 #define WEBRTC_PREAMP_GAIN_MAX_DB 90
61
62 static const char* const valid_modargs[] = {
63 "agc_start_volume",
64 "analog_gain_control",
65 "digital_gain_control",
66 "high_pass_filter",
67 "mobile",
68 "noise_suppression",
69 "post_amplifier",
70 "post_amplifier_gain",
71 "pre_amplifier",
72 "pre_amplifier_gain",
73 "transient_noise_suppression",
74 "voice_detection",
75 NULL
76 };
77
webrtc_volume_from_pa(pa_volume_t v)78 static int webrtc_volume_from_pa(pa_volume_t v)
79 {
80 return (v * WEBRTC_AGC_MAX_VOLUME) / PA_VOLUME_NORM;
81 }
82
webrtc_volume_to_pa(int v)83 static pa_volume_t webrtc_volume_to_pa(int v)
84 {
85 return (v * PA_VOLUME_NORM) / WEBRTC_AGC_MAX_VOLUME;
86 }
87
webrtc_ec_fixate_spec(pa_sample_spec * rec_ss,pa_channel_map * rec_map,pa_sample_spec * play_ss,pa_channel_map * play_map,pa_sample_spec * out_ss,pa_channel_map * out_map)88 static void webrtc_ec_fixate_spec(pa_sample_spec *rec_ss, pa_channel_map *rec_map,
89 pa_sample_spec *play_ss, pa_channel_map *play_map,
90 pa_sample_spec *out_ss, pa_channel_map *out_map)
91 {
92 rec_ss->format = PA_SAMPLE_FLOAT32NE;
93 play_ss->format = PA_SAMPLE_FLOAT32NE;
94
95 /* AudioProcessing expects one of the following rates */
96 if (rec_ss->rate >= 48000)
97 rec_ss->rate = 48000;
98 else if (rec_ss->rate >= 32000)
99 rec_ss->rate = 32000;
100 else if (rec_ss->rate >= 16000)
101 rec_ss->rate = 16000;
102 else
103 rec_ss->rate = 8000;
104
105 *out_ss = *rec_ss;
106 *out_map = *rec_map;
107
108 /* Playback stream rate needs to be the same as capture */
109 play_ss->rate = rec_ss->rate;
110 }
111
pa_webrtc_ec_init(pa_core * c,pa_echo_canceller * ec,pa_sample_spec * rec_ss,pa_channel_map * rec_map,pa_sample_spec * play_ss,pa_channel_map * play_map,pa_sample_spec * out_ss,pa_channel_map * out_map,uint32_t * nframes,const char * args)112 bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
113 pa_sample_spec *rec_ss, pa_channel_map *rec_map,
114 pa_sample_spec *play_ss, pa_channel_map *play_map,
115 pa_sample_spec *out_ss, pa_channel_map *out_map,
116 uint32_t *nframes, const char *args) {
117 webrtc::AudioProcessing *apm = webrtc::AudioProcessingBuilder().Create();
118 webrtc::ProcessingConfig pconfig;
119 webrtc::AudioProcessing::Config config;
120 bool hpf, ns, tns, agc, dgc, mobile, pre_amp, vad, post_amp;
121 int i;
122 uint32_t agc_start_volume, pre_amp_gain, post_amp_gain;
123 pa_modargs *ma;
124
125 if (!(ma = pa_modargs_new(args, valid_modargs))) {
126 pa_log("Failed to parse submodule arguments.");
127 goto fail;
128 }
129
130 hpf = DEFAULT_HIGH_PASS_FILTER;
131 if (pa_modargs_get_value_boolean(ma, "high_pass_filter", &hpf) < 0) {
132 pa_log("Failed to parse high_pass_filter value");
133 goto fail;
134 }
135
136 ns = DEFAULT_NOISE_SUPPRESSION;
137 if (pa_modargs_get_value_boolean(ma, "noise_suppression", &ns) < 0) {
138 pa_log("Failed to parse noise_suppression value");
139 goto fail;
140 }
141
142 tns = DEFAULT_TRANSIENT_NOISE_SUPPRESSION;
143 if (pa_modargs_get_value_boolean(ma, "transient_noise_suppression", &tns) < 0) {
144 pa_log("Failed to parse transient_noise_suppression value");
145 goto fail;
146 }
147
148 agc = DEFAULT_ANALOG_GAIN_CONTROL;
149 if (pa_modargs_get_value_boolean(ma, "analog_gain_control", &agc) < 0) {
150 pa_log("Failed to parse analog_gain_control value");
151 goto fail;
152 }
153
154 dgc = agc ? false : DEFAULT_DIGITAL_GAIN_CONTROL;
155 if (pa_modargs_get_value_boolean(ma, "digital_gain_control", &dgc) < 0) {
156 pa_log("Failed to parse digital_gain_control value");
157 goto fail;
158 }
159
160 if (agc && dgc) {
161 pa_log("You must pick only one between analog and digital gain control");
162 goto fail;
163 }
164
165 pre_amp = DEFAULT_PREAMP_ENABLE;
166 if (pa_modargs_get_value_boolean(ma, "pre_amplifier", &pre_amp) < 0) {
167 pa_log("Failed to parse pre_amplifier value");
168 goto fail;
169 }
170 pre_amp_gain = DEFAULT_PREAMP_GAIN_DB;
171 if (pa_modargs_get_value_u32(ma, "pre_amplifier_gain", &pre_amp_gain) < 0) {
172 pa_log("Failed to parse pre_amplifier_gain value");
173 goto fail;
174 }
175 if (pre_amp_gain > WEBRTC_PREAMP_GAIN_MAX_DB) {
176 pa_log("Preamp gain must not exceed %u", WEBRTC_PREAMP_GAIN_MAX_DB);
177 goto fail;
178 }
179
180 post_amp = DEFAULT_POSTAMP_ENABLE;
181 if (pa_modargs_get_value_boolean(ma, "post_amplifier", &post_amp) < 0) {
182 pa_log("Failed to parse post_amplifier value");
183 goto fail;
184 }
185 post_amp_gain = DEFAULT_POSTAMP_GAIN_DB;
186 if (pa_modargs_get_value_u32(ma, "post_amplifier_gain", &post_amp_gain) < 0) {
187 pa_log("Failed to parse post_amplifier_gain value");
188 goto fail;
189 }
190 if (post_amp_gain > WEBRTC_POSTAMP_GAIN_MAX_DB) {
191 pa_log("Postamp gain must not exceed %u", WEBRTC_POSTAMP_GAIN_MAX_DB);
192 goto fail;
193 }
194
195 mobile = DEFAULT_MOBILE;
196 if (pa_modargs_get_value_boolean(ma, "mobile", &mobile) < 0) {
197 pa_log("Failed to parse mobile value");
198 goto fail;
199 }
200
201 ec->params.drift_compensation = DEFAULT_DRIFT_COMPENSATION;
202
203 vad = DEFAULT_VAD;
204 if (pa_modargs_get_value_boolean(ma, "voice_detection", &vad) < 0) {
205 pa_log("Failed to parse voice_detection value");
206 goto fail;
207 }
208
209 agc_start_volume = DEFAULT_AGC_START_VOLUME;
210 if (pa_modargs_get_value_u32(ma, "agc_start_volume", &agc_start_volume) < 0) {
211 pa_log("Failed to parse agc_start_volume value");
212 goto fail;
213 }
214 if (agc_start_volume > WEBRTC_AGC_MAX_VOLUME) {
215 pa_log("AGC start volume must not exceed %u", WEBRTC_AGC_MAX_VOLUME);
216 goto fail;
217 }
218 ec->params.webrtc.agc_start_volume = agc_start_volume;
219
220 webrtc_ec_fixate_spec(rec_ss, rec_map, play_ss, play_map, out_ss, out_map);
221
222 pconfig = {
223 webrtc::StreamConfig(rec_ss->rate, rec_ss->channels, false), /* input stream */
224 webrtc::StreamConfig(out_ss->rate, out_ss->channels, false), /* output stream */
225 webrtc::StreamConfig(play_ss->rate, play_ss->channels, false), /* reverse input stream */
226 webrtc::StreamConfig(play_ss->rate, play_ss->channels, false), /* reverse output stream */
227 };
228 if (apm->Initialize(pconfig) != webrtc::AudioProcessing::kNoError) {
229 pa_log("Error initialising audio processing module");
230 goto fail;
231 }
232
233 if (pre_amp) {
234 config.pre_amplifier.enabled = true;
235 config.pre_amplifier.fixed_gain_factor = (float)pre_amp_gain;
236 } else
237 config.pre_amplifier.enabled = false;
238
239 if (hpf)
240 config.high_pass_filter.enabled = true;
241 else
242 config.high_pass_filter.enabled = false;
243
244 config.echo_canceller.enabled = true;
245
246 if (!mobile)
247 config.echo_canceller.mobile_mode = false;
248 else
249 config.echo_canceller.mobile_mode = true;
250
251 if (ns)
252 config.noise_suppression.enabled = true;
253 else
254 config.noise_suppression.enabled = false;
255
256 if (tns)
257 config.transient_suppression.enabled = true;
258 else
259 config.transient_suppression.enabled = false;
260
261 if (dgc) {
262 ec->params.webrtc.agc = false;
263 config.gain_controller1.enabled = true;
264 if (mobile)
265 config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kFixedDigital;
266 else
267 config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveDigital;
268 } else if (agc) {
269 ec->params.webrtc.agc = true;
270 config.gain_controller1.enabled = true;
271 config.gain_controller1.mode = webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog;
272 config.gain_controller1.analog_level_minimum = 0;
273 config.gain_controller1.analog_level_maximum = WEBRTC_AGC_MAX_VOLUME;
274 }
275
276 if (vad)
277 config.voice_detection.enabled = true;
278 else
279 config.voice_detection.enabled = false;
280
281 if (post_amp) {
282 config.gain_controller2.enabled = true;
283 config.gain_controller2.fixed_digital.gain_db = (float)post_amp_gain;
284 config.gain_controller2.adaptive_digital.enabled = false;
285 } else
286 config.gain_controller2.enabled = false;
287
288 ec->params.webrtc.apm = apm;
289 ec->params.webrtc.rec_ss = *rec_ss;
290 ec->params.webrtc.play_ss = *play_ss;
291 ec->params.webrtc.out_ss = *out_ss;
292 ec->params.webrtc.blocksize = (uint64_t) out_ss->rate * BLOCK_SIZE_US / PA_USEC_PER_SEC;
293 *nframes = ec->params.webrtc.blocksize;
294 ec->params.webrtc.first = true;
295
296 apm->ApplyConfig(config);
297
298 for (i = 0; i < rec_ss->channels; i++)
299 ec->params.webrtc.rec_buffer[i] = pa_xnew(float, *nframes);
300 for (i = 0; i < play_ss->channels; i++)
301 ec->params.webrtc.play_buffer[i] = pa_xnew(float, *nframes);
302
303 pa_modargs_free(ma);
304 return true;
305
306 fail:
307 if (ma)
308 pa_modargs_free(ma);
309 if (apm)
310 delete apm;
311
312 return false;
313 }
314
pa_webrtc_ec_play(pa_echo_canceller * ec,const uint8_t * play)315 void pa_webrtc_ec_play(pa_echo_canceller *ec, const uint8_t *play) {
316 webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
317 const pa_sample_spec *ss = &ec->params.webrtc.play_ss;
318 int n = ec->params.webrtc.blocksize;
319 float **buf = ec->params.webrtc.play_buffer;
320 webrtc::StreamConfig config(ss->rate, ss->channels, false);
321
322 pa_deinterleave(play, (void **) buf, ss->channels, pa_sample_size(ss), n);
323
324 pa_assert_se(apm->ProcessReverseStream(buf, config, config, buf) == webrtc::AudioProcessing::kNoError);
325 }
326
pa_webrtc_ec_record(pa_echo_canceller * ec,const uint8_t * rec,uint8_t * out)327 void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out) {
328 webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
329 const pa_sample_spec *rec_ss = &ec->params.webrtc.rec_ss;
330 const pa_sample_spec *out_ss = &ec->params.webrtc.out_ss;
331 float **buf = ec->params.webrtc.rec_buffer;
332 int n = ec->params.webrtc.blocksize;
333 int old_volume, new_volume;
334 webrtc::StreamConfig rec_config(rec_ss->rate, rec_ss->channels, false);
335 webrtc::StreamConfig out_config(out_ss->rate, out_ss->channels, false);
336
337 pa_deinterleave(rec, (void **) buf, rec_ss->channels, pa_sample_size(rec_ss), n);
338
339 if (ec->params.webrtc.agc) {
340 pa_volume_t v = pa_echo_canceller_get_capture_volume(ec);
341 old_volume = webrtc_volume_from_pa(v);
342 apm->set_stream_analog_level(old_volume);
343 }
344
345 apm->set_stream_delay_ms(0);
346 pa_assert_se(apm->ProcessStream(buf, rec_config, out_config, buf) == webrtc::AudioProcessing::kNoError);
347
348 if (ec->params.webrtc.agc) {
349 if (PA_UNLIKELY(ec->params.webrtc.first)) {
350 /* We start at a sane default volume (taken from the Chromium
351 * condition on the experimental AGC in audio_processing.h). This is
352 * needed to make sure that there's enough energy in the capture
353 * signal for the AGC to work */
354 ec->params.webrtc.first = false;
355 new_volume = ec->params.webrtc.agc_start_volume;
356 } else {
357 new_volume = apm->recommended_stream_analog_level();
358 }
359
360 if (old_volume != new_volume)
361 pa_echo_canceller_set_capture_volume(ec, webrtc_volume_to_pa(new_volume));
362 }
363
364 pa_interleave((const void **) buf, out_ss->channels, out, pa_sample_size(out_ss), n);
365 }
366
pa_webrtc_ec_set_drift(pa_echo_canceller * ec,float drift)367 void pa_webrtc_ec_set_drift(pa_echo_canceller *ec, float drift) {
368 }
369
pa_webrtc_ec_run(pa_echo_canceller * ec,const uint8_t * rec,const uint8_t * play,uint8_t * out)370 void pa_webrtc_ec_run(pa_echo_canceller *ec, const uint8_t *rec, const uint8_t *play, uint8_t *out) {
371 pa_webrtc_ec_play(ec, play);
372 pa_webrtc_ec_record(ec, rec, out);
373 }
374
pa_webrtc_ec_done(pa_echo_canceller * ec)375 void pa_webrtc_ec_done(pa_echo_canceller *ec) {
376 int i;
377
378 if (ec->params.webrtc.apm) {
379 delete (webrtc::AudioProcessing*)ec->params.webrtc.apm;
380 ec->params.webrtc.apm = NULL;
381 }
382
383 for (i = 0; i < ec->params.webrtc.rec_ss.channels; i++)
384 pa_xfree(ec->params.webrtc.rec_buffer[i]);
385 for (i = 0; i < ec->params.webrtc.play_ss.channels; i++)
386 pa_xfree(ec->params.webrtc.play_buffer[i]);
387 }
388