1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h"
12 #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h"
13 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
14 #include "webrtc/modules/audio_processing/include/audio_processing.h"
15 #include "webrtc/modules/utility/include/audio_frame_operations.h"
16 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
17 #include "webrtc/system_wrappers/include/trace.h"
18
19 namespace webrtc {
20 namespace {
21
22 struct ParticipantFramePair {
23 MixerParticipant* participant;
24 AudioFrame* audioFrame;
25 };
26
27 typedef std::list<ParticipantFramePair*> ParticipantFramePairList;
28
29 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
30 // These effects are applied to |frame| itself prior to mixing. Assumes that
31 // |mixed_frame| always has at least as many channels as |frame|. Supports
32 // stereo at most.
33 //
34 // TODO(andrew): consider not modifying |frame| here.
MixFrames(AudioFrame * mixed_frame,AudioFrame * frame,bool use_limiter)35 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) {
36 assert(mixed_frame->num_channels_ >= frame->num_channels_);
37 if (use_limiter) {
38 // Divide by two to avoid saturation in the mixing.
39 // This is only meaningful if the limiter will be used.
40 *frame >>= 1;
41 }
42 if (mixed_frame->num_channels_ > frame->num_channels_) {
43 // We only support mono-to-stereo.
44 assert(mixed_frame->num_channels_ == 2 &&
45 frame->num_channels_ == 1);
46 AudioFrameOperations::MonoToStereo(frame);
47 }
48
49 *mixed_frame += *frame;
50 }
51
52 // Return the max number of channels from a |list| composed of AudioFrames.
MaxNumChannels(const AudioFrameList * list)53 size_t MaxNumChannels(const AudioFrameList* list) {
54 size_t max_num_channels = 1;
55 for (AudioFrameList::const_iterator iter = list->begin();
56 iter != list->end();
57 ++iter) {
58 max_num_channels = std::max(max_num_channels, (*iter)->num_channels_);
59 }
60 return max_num_channels;
61 }
62
63 } // namespace
64
MixerParticipant()65 MixerParticipant::MixerParticipant()
66 : _mixHistory(new MixHistory()) {
67 }
68
~MixerParticipant()69 MixerParticipant::~MixerParticipant() {
70 delete _mixHistory;
71 }
72
IsMixed() const73 bool MixerParticipant::IsMixed() const {
74 return _mixHistory->IsMixed();
75 }
76
MixHistory()77 MixHistory::MixHistory()
78 : _isMixed(0) {
79 }
80
~MixHistory()81 MixHistory::~MixHistory() {
82 }
83
IsMixed() const84 bool MixHistory::IsMixed() const {
85 return _isMixed;
86 }
87
WasMixed() const88 bool MixHistory::WasMixed() const {
89 // Was mixed is the same as is mixed depending on perspective. This function
90 // is for the perspective of AudioConferenceMixerImpl.
91 return IsMixed();
92 }
93
SetIsMixed(const bool mixed)94 int32_t MixHistory::SetIsMixed(const bool mixed) {
95 _isMixed = mixed;
96 return 0;
97 }
98
ResetMixedStatus()99 void MixHistory::ResetMixedStatus() {
100 _isMixed = false;
101 }
102
Create(int id)103 AudioConferenceMixer* AudioConferenceMixer::Create(int id) {
104 AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id);
105 if(!mixer->Init()) {
106 delete mixer;
107 return NULL;
108 }
109 return mixer;
110 }
111
AudioConferenceMixerImpl(int id)112 AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id)
113 : _id(id),
114 _minimumMixingFreq(kLowestPossible),
115 _mixReceiver(NULL),
116 _outputFrequency(kDefaultFrequency),
117 _sampleSize(0),
118 _audioFramePool(NULL),
119 _participantList(),
120 _additionalParticipantList(),
121 _numMixedParticipants(0),
122 use_limiter_(true),
123 _timeStamp(0),
124 _timeScheduler(kProcessPeriodicityInMs),
125 _processCalls(0) {}
126
Init()127 bool AudioConferenceMixerImpl::Init() {
128 _crit.reset(CriticalSectionWrapper::CreateCriticalSection());
129 if (_crit.get() == NULL)
130 return false;
131
132 _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection());
133 if(_cbCrit.get() == NULL)
134 return false;
135
136 Config config;
137 config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
138 _limiter.reset(AudioProcessing::Create(config));
139 if(!_limiter.get())
140 return false;
141
142 MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool,
143 DEFAULT_AUDIO_FRAME_POOLSIZE);
144 if(_audioFramePool == NULL)
145 return false;
146
147 if(SetOutputFrequency(kDefaultFrequency) == -1)
148 return false;
149
150 if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
151 _limiter->kNoError)
152 return false;
153
154 // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
155 // divide-by-2 but -7 is used instead to give a bit of headroom since the
156 // AGC is not a hard limiter.
157 if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError)
158 return false;
159
160 if(_limiter->gain_control()->set_compression_gain_db(0)
161 != _limiter->kNoError)
162 return false;
163
164 if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
165 return false;
166
167 if(_limiter->gain_control()->Enable(true) != _limiter->kNoError)
168 return false;
169
170 return true;
171 }
172
~AudioConferenceMixerImpl()173 AudioConferenceMixerImpl::~AudioConferenceMixerImpl() {
174 MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool);
175 assert(_audioFramePool == NULL);
176 }
177
178 // Process should be called every kProcessPeriodicityInMs ms
TimeUntilNextProcess()179 int64_t AudioConferenceMixerImpl::TimeUntilNextProcess() {
180 int64_t timeUntilNextProcess = 0;
181 CriticalSectionScoped cs(_crit.get());
182 if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) {
183 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
184 "failed in TimeToNextUpdate() call");
185 // Sanity check
186 assert(false);
187 return -1;
188 }
189 return timeUntilNextProcess;
190 }
191
Process()192 int32_t AudioConferenceMixerImpl::Process() {
193 size_t remainingParticipantsAllowedToMix =
194 kMaximumAmountOfMixedParticipants;
195 {
196 CriticalSectionScoped cs(_crit.get());
197 assert(_processCalls == 0);
198 _processCalls++;
199
200 // Let the scheduler know that we are running one iteration.
201 _timeScheduler.UpdateScheduler();
202 }
203
204 AudioFrameList mixList;
205 AudioFrameList rampOutList;
206 AudioFrameList additionalFramesList;
207 std::map<int, MixerParticipant*> mixedParticipantsMap;
208 {
209 CriticalSectionScoped cs(_cbCrit.get());
210
211 int32_t lowFreq = GetLowestMixingFrequency();
212 // SILK can run in 12 kHz and 24 kHz. These frequencies are not
213 // supported so use the closest higher frequency to not lose any
214 // information.
215 // TODO(henrike): this is probably more appropriate to do in
216 // GetLowestMixingFrequency().
217 if (lowFreq == 12000) {
218 lowFreq = 16000;
219 } else if (lowFreq == 24000) {
220 lowFreq = 32000;
221 }
222 if(lowFreq <= 0) {
223 CriticalSectionScoped cs(_crit.get());
224 _processCalls--;
225 return 0;
226 } else {
227 switch(lowFreq) {
228 case 8000:
229 if(OutputFrequency() != kNbInHz) {
230 SetOutputFrequency(kNbInHz);
231 }
232 break;
233 case 16000:
234 if(OutputFrequency() != kWbInHz) {
235 SetOutputFrequency(kWbInHz);
236 }
237 break;
238 case 32000:
239 if(OutputFrequency() != kSwbInHz) {
240 SetOutputFrequency(kSwbInHz);
241 }
242 break;
243 case 48000:
244 if(OutputFrequency() != kFbInHz) {
245 SetOutputFrequency(kFbInHz);
246 }
247 break;
248 default:
249 assert(false);
250
251 CriticalSectionScoped cs(_crit.get());
252 _processCalls--;
253 return -1;
254 }
255 }
256
257 UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap,
258 &remainingParticipantsAllowedToMix);
259
260 GetAdditionalAudio(&additionalFramesList);
261 UpdateMixedStatus(mixedParticipantsMap);
262 }
263
264 // Get an AudioFrame for mixing from the memory pool.
265 AudioFrame* mixedAudio = NULL;
266 if(_audioFramePool->PopMemory(mixedAudio) == -1) {
267 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
268 "failed PopMemory() call");
269 assert(false);
270 return -1;
271 }
272
273 int retval = 0;
274 {
275 CriticalSectionScoped cs(_crit.get());
276
277 // TODO(henrike): it might be better to decide the number of channels
278 // with an API instead of dynamically.
279
280 // Find the max channels over all mixing lists.
281 const size_t num_mixed_channels = std::max(MaxNumChannels(&mixList),
282 std::max(MaxNumChannels(&additionalFramesList),
283 MaxNumChannels(&rampOutList)));
284
285 mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency,
286 AudioFrame::kNormalSpeech,
287 AudioFrame::kVadPassive, num_mixed_channels);
288
289 _timeStamp += static_cast<uint32_t>(_sampleSize);
290
291 // We only use the limiter if it supports the output sample rate and
292 // we're actually mixing multiple streams.
293 use_limiter_ =
294 _numMixedParticipants > 1 &&
295 _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz;
296
297 MixFromList(mixedAudio, mixList);
298 MixAnonomouslyFromList(mixedAudio, additionalFramesList);
299 MixAnonomouslyFromList(mixedAudio, rampOutList);
300
301 if(mixedAudio->samples_per_channel_ == 0) {
302 // Nothing was mixed, set the audio samples to silence.
303 mixedAudio->samples_per_channel_ = _sampleSize;
304 mixedAudio->Mute();
305 } else {
306 // Only call the limiter if we have something to mix.
307 if(!LimitMixedAudio(mixedAudio))
308 retval = -1;
309 }
310 }
311
312 {
313 CriticalSectionScoped cs(_cbCrit.get());
314 if(_mixReceiver != NULL) {
315 const AudioFrame** dummy = NULL;
316 _mixReceiver->NewMixedAudio(
317 _id,
318 *mixedAudio,
319 dummy,
320 0);
321 }
322 }
323
324 // Reclaim all outstanding memory.
325 _audioFramePool->PushMemory(mixedAudio);
326 ClearAudioFrameList(&mixList);
327 ClearAudioFrameList(&rampOutList);
328 ClearAudioFrameList(&additionalFramesList);
329 {
330 CriticalSectionScoped cs(_crit.get());
331 _processCalls--;
332 }
333 return retval;
334 }
335
RegisterMixedStreamCallback(AudioMixerOutputReceiver * mixReceiver)336 int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback(
337 AudioMixerOutputReceiver* mixReceiver) {
338 CriticalSectionScoped cs(_cbCrit.get());
339 if(_mixReceiver != NULL) {
340 return -1;
341 }
342 _mixReceiver = mixReceiver;
343 return 0;
344 }
345
UnRegisterMixedStreamCallback()346 int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() {
347 CriticalSectionScoped cs(_cbCrit.get());
348 if(_mixReceiver == NULL) {
349 return -1;
350 }
351 _mixReceiver = NULL;
352 return 0;
353 }
354
SetOutputFrequency(const Frequency & frequency)355 int32_t AudioConferenceMixerImpl::SetOutputFrequency(
356 const Frequency& frequency) {
357 CriticalSectionScoped cs(_crit.get());
358
359 _outputFrequency = frequency;
360 _sampleSize =
361 static_cast<size_t>((_outputFrequency*kProcessPeriodicityInMs) / 1000);
362
363 return 0;
364 }
365
366 AudioConferenceMixer::Frequency
OutputFrequency() const367 AudioConferenceMixerImpl::OutputFrequency() const {
368 CriticalSectionScoped cs(_crit.get());
369 return _outputFrequency;
370 }
371
SetMixabilityStatus(MixerParticipant * participant,bool mixable)372 int32_t AudioConferenceMixerImpl::SetMixabilityStatus(
373 MixerParticipant* participant, bool mixable) {
374 if (!mixable) {
375 // Anonymous participants are in a separate list. Make sure that the
376 // participant is in the _participantList if it is being mixed.
377 SetAnonymousMixabilityStatus(participant, false);
378 }
379 size_t numMixedParticipants;
380 {
381 CriticalSectionScoped cs(_cbCrit.get());
382 const bool isMixed =
383 IsParticipantInList(*participant, _participantList);
384 // API must be called with a new state.
385 if(!(mixable ^ isMixed)) {
386 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
387 "Mixable is aready %s",
388 isMixed ? "ON" : "off");
389 return -1;
390 }
391 bool success = false;
392 if(mixable) {
393 success = AddParticipantToList(participant, &_participantList);
394 } else {
395 success = RemoveParticipantFromList(participant, &_participantList);
396 }
397 if(!success) {
398 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
399 "failed to %s participant",
400 mixable ? "add" : "remove");
401 assert(false);
402 return -1;
403 }
404
405 size_t numMixedNonAnonymous = _participantList.size();
406 if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) {
407 numMixedNonAnonymous = kMaximumAmountOfMixedParticipants;
408 }
409 numMixedParticipants =
410 numMixedNonAnonymous + _additionalParticipantList.size();
411 }
412 // A MixerParticipant was added or removed. Make sure the scratch
413 // buffer is updated if necessary.
414 // Note: The scratch buffer may only be updated in Process().
415 CriticalSectionScoped cs(_crit.get());
416 _numMixedParticipants = numMixedParticipants;
417 return 0;
418 }
419
MixabilityStatus(const MixerParticipant & participant) const420 bool AudioConferenceMixerImpl::MixabilityStatus(
421 const MixerParticipant& participant) const {
422 CriticalSectionScoped cs(_cbCrit.get());
423 return IsParticipantInList(participant, _participantList);
424 }
425
SetAnonymousMixabilityStatus(MixerParticipant * participant,bool anonymous)426 int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus(
427 MixerParticipant* participant, bool anonymous) {
428 CriticalSectionScoped cs(_cbCrit.get());
429 if(IsParticipantInList(*participant, _additionalParticipantList)) {
430 if(anonymous) {
431 return 0;
432 }
433 if(!RemoveParticipantFromList(participant,
434 &_additionalParticipantList)) {
435 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
436 "unable to remove participant from anonymous list");
437 assert(false);
438 return -1;
439 }
440 return AddParticipantToList(participant, &_participantList) ? 0 : -1;
441 }
442 if(!anonymous) {
443 return 0;
444 }
445 const bool mixable = RemoveParticipantFromList(participant,
446 &_participantList);
447 if(!mixable) {
448 WEBRTC_TRACE(
449 kTraceWarning,
450 kTraceAudioMixerServer,
451 _id,
452 "participant must be registered before turning it into anonymous");
453 // Setting anonymous status is only possible if MixerParticipant is
454 // already registered.
455 return -1;
456 }
457 return AddParticipantToList(participant, &_additionalParticipantList) ?
458 0 : -1;
459 }
460
AnonymousMixabilityStatus(const MixerParticipant & participant) const461 bool AudioConferenceMixerImpl::AnonymousMixabilityStatus(
462 const MixerParticipant& participant) const {
463 CriticalSectionScoped cs(_cbCrit.get());
464 return IsParticipantInList(participant, _additionalParticipantList);
465 }
466
SetMinimumMixingFrequency(Frequency freq)467 int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency(
468 Frequency freq) {
469 // Make sure that only allowed sampling frequencies are used. Use closest
470 // higher sampling frequency to avoid losing information.
471 if (static_cast<int>(freq) == 12000) {
472 freq = kWbInHz;
473 } else if (static_cast<int>(freq) == 24000) {
474 freq = kSwbInHz;
475 }
476
477 if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) ||
478 (freq == kLowestPossible)) {
479 _minimumMixingFreq=freq;
480 return 0;
481 } else {
482 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
483 "SetMinimumMixingFrequency incorrect frequency: %i",freq);
484 assert(false);
485 return -1;
486 }
487 }
488
489 // Check all AudioFrames that are to be mixed. The highest sampling frequency
490 // found is the lowest that can be used without losing information.
GetLowestMixingFrequency() const491 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() const {
492 const int participantListFrequency =
493 GetLowestMixingFrequencyFromList(_participantList);
494 const int anonymousListFrequency =
495 GetLowestMixingFrequencyFromList(_additionalParticipantList);
496 const int highestFreq =
497 (participantListFrequency > anonymousListFrequency) ?
498 participantListFrequency : anonymousListFrequency;
499 // Check if the user specified a lowest mixing frequency.
500 if(_minimumMixingFreq != kLowestPossible) {
501 if(_minimumMixingFreq > highestFreq) {
502 return _minimumMixingFreq;
503 }
504 }
505 return highestFreq;
506 }
507
GetLowestMixingFrequencyFromList(const MixerParticipantList & mixList) const508 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
509 const MixerParticipantList& mixList) const {
510 int32_t highestFreq = 8000;
511 for (MixerParticipantList::const_iterator iter = mixList.begin();
512 iter != mixList.end();
513 ++iter) {
514 const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
515 if(neededFrequency > highestFreq) {
516 highestFreq = neededFrequency;
517 }
518 }
519 return highestFreq;
520 }
521
UpdateToMix(AudioFrameList * mixList,AudioFrameList * rampOutList,std::map<int,MixerParticipant * > * mixParticipantList,size_t * maxAudioFrameCounter) const522 void AudioConferenceMixerImpl::UpdateToMix(
523 AudioFrameList* mixList,
524 AudioFrameList* rampOutList,
525 std::map<int, MixerParticipant*>* mixParticipantList,
526 size_t* maxAudioFrameCounter) const {
527 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
528 "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)",
529 *maxAudioFrameCounter);
530 const size_t mixListStartSize = mixList->size();
531 AudioFrameList activeList;
532 // Struct needed by the passive lists to keep track of which AudioFrame
533 // belongs to which MixerParticipant.
534 ParticipantFramePairList passiveWasNotMixedList;
535 ParticipantFramePairList passiveWasMixedList;
536 for (MixerParticipantList::const_iterator participant =
537 _participantList.begin(); participant != _participantList.end();
538 ++participant) {
539 // Stop keeping track of passive participants if there are already
540 // enough participants available (they wont be mixed anyway).
541 bool mustAddToPassiveList = (*maxAudioFrameCounter >
542 (activeList.size() +
543 passiveWasMixedList.size() +
544 passiveWasNotMixedList.size()));
545
546 bool wasMixed = false;
547 wasMixed = (*participant)->_mixHistory->WasMixed();
548 AudioFrame* audioFrame = NULL;
549 if(_audioFramePool->PopMemory(audioFrame) == -1) {
550 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
551 "failed PopMemory() call");
552 assert(false);
553 return;
554 }
555 audioFrame->sample_rate_hz_ = _outputFrequency;
556
557 if((*participant)->GetAudioFrame(_id, audioFrame) != 0) {
558 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
559 "failed to GetAudioFrame() from participant");
560 _audioFramePool->PushMemory(audioFrame);
561 continue;
562 }
563 if (_participantList.size() != 1) {
564 // TODO(wu): Issue 3390, add support for multiple participants case.
565 audioFrame->ntp_time_ms_ = -1;
566 }
567
568 // TODO(henrike): this assert triggers in some test cases where SRTP is
569 // used which prevents NetEQ from making a VAD. Temporarily disable this
570 // assert until the problem is fixed on a higher level.
571 // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown);
572 if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) {
573 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
574 "invalid VAD state from participant");
575 }
576
577 if(audioFrame->vad_activity_ == AudioFrame::kVadActive) {
578 if(!wasMixed) {
579 RampIn(*audioFrame);
580 }
581
582 if(activeList.size() >= *maxAudioFrameCounter) {
583 // There are already more active participants than should be
584 // mixed. Only keep the ones with the highest energy.
585 AudioFrameList::iterator replaceItem;
586 CalculateEnergy(*audioFrame);
587 uint32_t lowestEnergy = audioFrame->energy_;
588
589 bool found_replace_item = false;
590 for (AudioFrameList::iterator iter = activeList.begin();
591 iter != activeList.end();
592 ++iter) {
593 CalculateEnergy(**iter);
594 if((*iter)->energy_ < lowestEnergy) {
595 replaceItem = iter;
596 lowestEnergy = (*iter)->energy_;
597 found_replace_item = true;
598 }
599 }
600 if(found_replace_item) {
601 AudioFrame* replaceFrame = *replaceItem;
602
603 bool replaceWasMixed = false;
604 std::map<int, MixerParticipant*>::const_iterator it =
605 mixParticipantList->find(replaceFrame->id_);
606
607 // When a frame is pushed to |activeList| it is also pushed
608 // to mixParticipantList with the frame's id. This means
609 // that the Find call above should never fail.
610 assert(it != mixParticipantList->end());
611 replaceWasMixed = it->second->_mixHistory->WasMixed();
612
613 mixParticipantList->erase(replaceFrame->id_);
614 activeList.erase(replaceItem);
615
616 activeList.push_front(audioFrame);
617 (*mixParticipantList)[audioFrame->id_] = *participant;
618 assert(mixParticipantList->size() <=
619 kMaximumAmountOfMixedParticipants);
620
621 if (replaceWasMixed) {
622 RampOut(*replaceFrame);
623 rampOutList->push_back(replaceFrame);
624 assert(rampOutList->size() <=
625 kMaximumAmountOfMixedParticipants);
626 } else {
627 _audioFramePool->PushMemory(replaceFrame);
628 }
629 } else {
630 if(wasMixed) {
631 RampOut(*audioFrame);
632 rampOutList->push_back(audioFrame);
633 assert(rampOutList->size() <=
634 kMaximumAmountOfMixedParticipants);
635 } else {
636 _audioFramePool->PushMemory(audioFrame);
637 }
638 }
639 } else {
640 activeList.push_front(audioFrame);
641 (*mixParticipantList)[audioFrame->id_] = *participant;
642 assert(mixParticipantList->size() <=
643 kMaximumAmountOfMixedParticipants);
644 }
645 } else {
646 if(wasMixed) {
647 ParticipantFramePair* pair = new ParticipantFramePair;
648 pair->audioFrame = audioFrame;
649 pair->participant = *participant;
650 passiveWasMixedList.push_back(pair);
651 } else if(mustAddToPassiveList) {
652 RampIn(*audioFrame);
653 ParticipantFramePair* pair = new ParticipantFramePair;
654 pair->audioFrame = audioFrame;
655 pair->participant = *participant;
656 passiveWasNotMixedList.push_back(pair);
657 } else {
658 _audioFramePool->PushMemory(audioFrame);
659 }
660 }
661 }
662 assert(activeList.size() <= *maxAudioFrameCounter);
663 // At this point it is known which participants should be mixed. Transfer
664 // this information to this functions output parameters.
665 for (AudioFrameList::const_iterator iter = activeList.begin();
666 iter != activeList.end();
667 ++iter) {
668 mixList->push_back(*iter);
669 }
670 activeList.clear();
671 // Always mix a constant number of AudioFrames. If there aren't enough
672 // active participants mix passive ones. Starting with those that was mixed
673 // last iteration.
674 for (ParticipantFramePairList::const_iterator
675 iter = passiveWasMixedList.begin(); iter != passiveWasMixedList.end();
676 ++iter) {
677 if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) {
678 mixList->push_back((*iter)->audioFrame);
679 (*mixParticipantList)[(*iter)->audioFrame->id_] =
680 (*iter)->participant;
681 assert(mixParticipantList->size() <=
682 kMaximumAmountOfMixedParticipants);
683 } else {
684 _audioFramePool->PushMemory((*iter)->audioFrame);
685 }
686 delete *iter;
687 }
688 // And finally the ones that have not been mixed for a while.
689 for (ParticipantFramePairList::const_iterator iter =
690 passiveWasNotMixedList.begin();
691 iter != passiveWasNotMixedList.end();
692 ++iter) {
693 if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) {
694 mixList->push_back((*iter)->audioFrame);
695 (*mixParticipantList)[(*iter)->audioFrame->id_] =
696 (*iter)->participant;
697 assert(mixParticipantList->size() <=
698 kMaximumAmountOfMixedParticipants);
699 } else {
700 _audioFramePool->PushMemory((*iter)->audioFrame);
701 }
702 delete *iter;
703 }
704 assert(*maxAudioFrameCounter + mixListStartSize >= mixList->size());
705 *maxAudioFrameCounter += mixListStartSize - mixList->size();
706 }
707
GetAdditionalAudio(AudioFrameList * additionalFramesList) const708 void AudioConferenceMixerImpl::GetAdditionalAudio(
709 AudioFrameList* additionalFramesList) const {
710 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
711 "GetAdditionalAudio(additionalFramesList)");
712 // The GetAudioFrame() callback may result in the participant being removed
713 // from additionalParticipantList_. If that happens it will invalidate any
714 // iterators. Create a copy of the participants list such that the list of
715 // participants can be traversed safely.
716 MixerParticipantList additionalParticipantList;
717 additionalParticipantList.insert(additionalParticipantList.begin(),
718 _additionalParticipantList.begin(),
719 _additionalParticipantList.end());
720
721 for (MixerParticipantList::const_iterator participant =
722 additionalParticipantList.begin();
723 participant != additionalParticipantList.end();
724 ++participant) {
725 AudioFrame* audioFrame = NULL;
726 if(_audioFramePool->PopMemory(audioFrame) == -1) {
727 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
728 "failed PopMemory() call");
729 assert(false);
730 return;
731 }
732 audioFrame->sample_rate_hz_ = _outputFrequency;
733 if((*participant)->GetAudioFrame(_id, audioFrame) != 0) {
734 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
735 "failed to GetAudioFrame() from participant");
736 _audioFramePool->PushMemory(audioFrame);
737 continue;
738 }
739 if(audioFrame->samples_per_channel_ == 0) {
740 // Empty frame. Don't use it.
741 _audioFramePool->PushMemory(audioFrame);
742 continue;
743 }
744 additionalFramesList->push_back(audioFrame);
745 }
746 }
747
UpdateMixedStatus(const std::map<int,MixerParticipant * > & mixedParticipantsMap) const748 void AudioConferenceMixerImpl::UpdateMixedStatus(
749 const std::map<int, MixerParticipant*>& mixedParticipantsMap) const {
750 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
751 "UpdateMixedStatus(mixedParticipantsMap)");
752 assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants);
753
754 // Loop through all participants. If they are in the mix map they
755 // were mixed.
756 for (MixerParticipantList::const_iterator
757 participant =_participantList.begin();
758 participant != _participantList.end();
759 ++participant) {
760 bool isMixed = false;
761 for (std::map<int, MixerParticipant*>::const_iterator it =
762 mixedParticipantsMap.begin();
763 it != mixedParticipantsMap.end();
764 ++it) {
765 if (it->second == *participant) {
766 isMixed = true;
767 break;
768 }
769 }
770 (*participant)->_mixHistory->SetIsMixed(isMixed);
771 }
772 }
773
ClearAudioFrameList(AudioFrameList * audioFrameList) const774 void AudioConferenceMixerImpl::ClearAudioFrameList(
775 AudioFrameList* audioFrameList) const {
776 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
777 "ClearAudioFrameList(audioFrameList)");
778 for (AudioFrameList::iterator iter = audioFrameList->begin();
779 iter != audioFrameList->end();
780 ++iter) {
781 _audioFramePool->PushMemory(*iter);
782 }
783 audioFrameList->clear();
784 }
785
UpdateVADPositiveParticipants(AudioFrameList * mixList) const786 void AudioConferenceMixerImpl::UpdateVADPositiveParticipants(
787 AudioFrameList* mixList) const {
788 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
789 "UpdateVADPositiveParticipants(mixList)");
790
791 for (AudioFrameList::const_iterator iter = mixList->begin();
792 iter != mixList->end();
793 ++iter) {
794 CalculateEnergy(**iter);
795 }
796 }
797
IsParticipantInList(const MixerParticipant & participant,const MixerParticipantList & participantList) const798 bool AudioConferenceMixerImpl::IsParticipantInList(
799 const MixerParticipant& participant,
800 const MixerParticipantList& participantList) const {
801 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
802 "IsParticipantInList(participant,participantList)");
803 for (MixerParticipantList::const_iterator iter = participantList.begin();
804 iter != participantList.end();
805 ++iter) {
806 if(&participant == *iter) {
807 return true;
808 }
809 }
810 return false;
811 }
812
AddParticipantToList(MixerParticipant * participant,MixerParticipantList * participantList) const813 bool AudioConferenceMixerImpl::AddParticipantToList(
814 MixerParticipant* participant,
815 MixerParticipantList* participantList) const {
816 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
817 "AddParticipantToList(participant, participantList)");
818 participantList->push_back(participant);
819 // Make sure that the mixed status is correct for new MixerParticipant.
820 participant->_mixHistory->ResetMixedStatus();
821 return true;
822 }
823
RemoveParticipantFromList(MixerParticipant * participant,MixerParticipantList * participantList) const824 bool AudioConferenceMixerImpl::RemoveParticipantFromList(
825 MixerParticipant* participant,
826 MixerParticipantList* participantList) const {
827 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
828 "RemoveParticipantFromList(participant, participantList)");
829 for (MixerParticipantList::iterator iter = participantList->begin();
830 iter != participantList->end();
831 ++iter) {
832 if(*iter == participant) {
833 participantList->erase(iter);
834 // Participant is no longer mixed, reset to default.
835 participant->_mixHistory->ResetMixedStatus();
836 return true;
837 }
838 }
839 return false;
840 }
841
MixFromList(AudioFrame * mixedAudio,const AudioFrameList & audioFrameList) const842 int32_t AudioConferenceMixerImpl::MixFromList(
843 AudioFrame* mixedAudio,
844 const AudioFrameList& audioFrameList) const {
845 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
846 "MixFromList(mixedAudio, audioFrameList)");
847 if(audioFrameList.empty()) return 0;
848
849 uint32_t position = 0;
850
851 if (_numMixedParticipants == 1) {
852 mixedAudio->timestamp_ = audioFrameList.front()->timestamp_;
853 mixedAudio->elapsed_time_ms_ = audioFrameList.front()->elapsed_time_ms_;
854 } else {
855 // TODO(wu): Issue 3390.
856 // Audio frame timestamp is only supported in one channel case.
857 mixedAudio->timestamp_ = 0;
858 mixedAudio->elapsed_time_ms_ = -1;
859 }
860
861 for (AudioFrameList::const_iterator iter = audioFrameList.begin();
862 iter != audioFrameList.end();
863 ++iter) {
864 if(position >= kMaximumAmountOfMixedParticipants) {
865 WEBRTC_TRACE(
866 kTraceMemory,
867 kTraceAudioMixerServer,
868 _id,
869 "Trying to mix more than max amount of mixed participants:%d!",
870 kMaximumAmountOfMixedParticipants);
871 // Assert and avoid crash
872 assert(false);
873 position = 0;
874 }
875 MixFrames(mixedAudio, (*iter), use_limiter_);
876
877 position++;
878 }
879
880 return 0;
881 }
882
883 // TODO(andrew): consolidate this function with MixFromList.
MixAnonomouslyFromList(AudioFrame * mixedAudio,const AudioFrameList & audioFrameList) const884 int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList(
885 AudioFrame* mixedAudio,
886 const AudioFrameList& audioFrameList) const {
887 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
888 "MixAnonomouslyFromList(mixedAudio, audioFrameList)");
889
890 if(audioFrameList.empty()) return 0;
891
892 for (AudioFrameList::const_iterator iter = audioFrameList.begin();
893 iter != audioFrameList.end();
894 ++iter) {
895 MixFrames(mixedAudio, *iter, use_limiter_);
896 }
897 return 0;
898 }
899
LimitMixedAudio(AudioFrame * mixedAudio) const900 bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame* mixedAudio) const {
901 if (!use_limiter_) {
902 return true;
903 }
904
905 // Smoothly limit the mixed frame.
906 const int error = _limiter->ProcessStream(mixedAudio);
907
908 // And now we can safely restore the level. This procedure results in
909 // some loss of resolution, deemed acceptable.
910 //
911 // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
912 // and compression gain of 6 dB). However, in the transition frame when this
913 // is enabled (moving from one to two participants) it has the potential to
914 // create discontinuities in the mixed frame.
915 //
916 // Instead we double the frame (with addition since left-shifting a
917 // negative value is undefined).
918 *mixedAudio += *mixedAudio;
919
920 if(error != _limiter->kNoError) {
921 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
922 "Error from AudioProcessing: %d", error);
923 assert(false);
924 return false;
925 }
926 return true;
927 }
928 } // namespace webrtc
929