• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h"
12 #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h"
13 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
14 #include "webrtc/modules/audio_processing/include/audio_processing.h"
15 #include "webrtc/modules/utility/include/audio_frame_operations.h"
16 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
17 #include "webrtc/system_wrappers/include/trace.h"
18 
19 namespace webrtc {
20 namespace {
21 
22 struct ParticipantFramePair {
23   MixerParticipant* participant;
24   AudioFrame* audioFrame;
25 };
26 
27 typedef std::list<ParticipantFramePair*> ParticipantFramePairList;
28 
29 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
30 // These effects are applied to |frame| itself prior to mixing. Assumes that
31 // |mixed_frame| always has at least as many channels as |frame|. Supports
32 // stereo at most.
33 //
34 // TODO(andrew): consider not modifying |frame| here.
MixFrames(AudioFrame * mixed_frame,AudioFrame * frame,bool use_limiter)35 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) {
36   assert(mixed_frame->num_channels_ >= frame->num_channels_);
37   if (use_limiter) {
38     // Divide by two to avoid saturation in the mixing.
39     // This is only meaningful if the limiter will be used.
40     *frame >>= 1;
41   }
42   if (mixed_frame->num_channels_ > frame->num_channels_) {
43     // We only support mono-to-stereo.
44     assert(mixed_frame->num_channels_ == 2 &&
45            frame->num_channels_ == 1);
46     AudioFrameOperations::MonoToStereo(frame);
47   }
48 
49   *mixed_frame += *frame;
50 }
51 
52 // Return the max number of channels from a |list| composed of AudioFrames.
MaxNumChannels(const AudioFrameList * list)53 size_t MaxNumChannels(const AudioFrameList* list) {
54   size_t max_num_channels = 1;
55   for (AudioFrameList::const_iterator iter = list->begin();
56        iter != list->end();
57        ++iter) {
58     max_num_channels = std::max(max_num_channels, (*iter)->num_channels_);
59   }
60   return max_num_channels;
61 }
62 
63 }  // namespace
64 
MixerParticipant()65 MixerParticipant::MixerParticipant()
66     : _mixHistory(new MixHistory()) {
67 }
68 
~MixerParticipant()69 MixerParticipant::~MixerParticipant() {
70     delete _mixHistory;
71 }
72 
IsMixed() const73 bool MixerParticipant::IsMixed() const {
74     return _mixHistory->IsMixed();
75 }
76 
MixHistory()77 MixHistory::MixHistory()
78     : _isMixed(0) {
79 }
80 
~MixHistory()81 MixHistory::~MixHistory() {
82 }
83 
IsMixed() const84 bool MixHistory::IsMixed() const {
85     return _isMixed;
86 }
87 
WasMixed() const88 bool MixHistory::WasMixed() const {
89     // Was mixed is the same as is mixed depending on perspective. This function
90     // is for the perspective of AudioConferenceMixerImpl.
91     return IsMixed();
92 }
93 
SetIsMixed(const bool mixed)94 int32_t MixHistory::SetIsMixed(const bool mixed) {
95     _isMixed = mixed;
96     return 0;
97 }
98 
ResetMixedStatus()99 void MixHistory::ResetMixedStatus() {
100     _isMixed = false;
101 }
102 
Create(int id)103 AudioConferenceMixer* AudioConferenceMixer::Create(int id) {
104     AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id);
105     if(!mixer->Init()) {
106         delete mixer;
107         return NULL;
108     }
109     return mixer;
110 }
111 
AudioConferenceMixerImpl(int id)112 AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id)
113     : _id(id),
114       _minimumMixingFreq(kLowestPossible),
115       _mixReceiver(NULL),
116       _outputFrequency(kDefaultFrequency),
117       _sampleSize(0),
118       _audioFramePool(NULL),
119       _participantList(),
120       _additionalParticipantList(),
121       _numMixedParticipants(0),
122       use_limiter_(true),
123       _timeStamp(0),
124       _timeScheduler(kProcessPeriodicityInMs),
125       _processCalls(0) {}
126 
Init()127 bool AudioConferenceMixerImpl::Init() {
128     _crit.reset(CriticalSectionWrapper::CreateCriticalSection());
129     if (_crit.get() == NULL)
130         return false;
131 
132     _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection());
133     if(_cbCrit.get() == NULL)
134         return false;
135 
136     Config config;
137     config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
138     _limiter.reset(AudioProcessing::Create(config));
139     if(!_limiter.get())
140         return false;
141 
142     MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool,
143                                              DEFAULT_AUDIO_FRAME_POOLSIZE);
144     if(_audioFramePool == NULL)
145         return false;
146 
147     if(SetOutputFrequency(kDefaultFrequency) == -1)
148         return false;
149 
150     if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
151         _limiter->kNoError)
152         return false;
153 
154     // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
155     // divide-by-2 but -7 is used instead to give a bit of headroom since the
156     // AGC is not a hard limiter.
157     if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError)
158         return false;
159 
160     if(_limiter->gain_control()->set_compression_gain_db(0)
161         != _limiter->kNoError)
162         return false;
163 
164     if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
165         return false;
166 
167     if(_limiter->gain_control()->Enable(true) != _limiter->kNoError)
168         return false;
169 
170     return true;
171 }
172 
~AudioConferenceMixerImpl()173 AudioConferenceMixerImpl::~AudioConferenceMixerImpl() {
174     MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool);
175     assert(_audioFramePool == NULL);
176 }
177 
178 // Process should be called every kProcessPeriodicityInMs ms
TimeUntilNextProcess()179 int64_t AudioConferenceMixerImpl::TimeUntilNextProcess() {
180     int64_t timeUntilNextProcess = 0;
181     CriticalSectionScoped cs(_crit.get());
182     if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) {
183         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
184                      "failed in TimeToNextUpdate() call");
185         // Sanity check
186         assert(false);
187         return -1;
188     }
189     return timeUntilNextProcess;
190 }
191 
Process()192 int32_t AudioConferenceMixerImpl::Process() {
193     size_t remainingParticipantsAllowedToMix =
194         kMaximumAmountOfMixedParticipants;
195     {
196         CriticalSectionScoped cs(_crit.get());
197         assert(_processCalls == 0);
198         _processCalls++;
199 
200         // Let the scheduler know that we are running one iteration.
201         _timeScheduler.UpdateScheduler();
202     }
203 
204     AudioFrameList mixList;
205     AudioFrameList rampOutList;
206     AudioFrameList additionalFramesList;
207     std::map<int, MixerParticipant*> mixedParticipantsMap;
208     {
209         CriticalSectionScoped cs(_cbCrit.get());
210 
211         int32_t lowFreq = GetLowestMixingFrequency();
212         // SILK can run in 12 kHz and 24 kHz. These frequencies are not
213         // supported so use the closest higher frequency to not lose any
214         // information.
215         // TODO(henrike): this is probably more appropriate to do in
216         //                GetLowestMixingFrequency().
217         if (lowFreq == 12000) {
218             lowFreq = 16000;
219         } else if (lowFreq == 24000) {
220             lowFreq = 32000;
221         }
222         if(lowFreq <= 0) {
223             CriticalSectionScoped cs(_crit.get());
224             _processCalls--;
225             return 0;
226         } else {
227             switch(lowFreq) {
228             case 8000:
229                 if(OutputFrequency() != kNbInHz) {
230                     SetOutputFrequency(kNbInHz);
231                 }
232                 break;
233             case 16000:
234                 if(OutputFrequency() != kWbInHz) {
235                     SetOutputFrequency(kWbInHz);
236                 }
237                 break;
238             case 32000:
239                 if(OutputFrequency() != kSwbInHz) {
240                     SetOutputFrequency(kSwbInHz);
241                 }
242                 break;
243             case 48000:
244                 if(OutputFrequency() != kFbInHz) {
245                     SetOutputFrequency(kFbInHz);
246                 }
247                 break;
248             default:
249                 assert(false);
250 
251                 CriticalSectionScoped cs(_crit.get());
252                 _processCalls--;
253                 return -1;
254             }
255         }
256 
257         UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap,
258                     &remainingParticipantsAllowedToMix);
259 
260         GetAdditionalAudio(&additionalFramesList);
261         UpdateMixedStatus(mixedParticipantsMap);
262     }
263 
264     // Get an AudioFrame for mixing from the memory pool.
265     AudioFrame* mixedAudio = NULL;
266     if(_audioFramePool->PopMemory(mixedAudio) == -1) {
267         WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
268                      "failed PopMemory() call");
269         assert(false);
270         return -1;
271     }
272 
273     int retval = 0;
274     {
275         CriticalSectionScoped cs(_crit.get());
276 
277         // TODO(henrike): it might be better to decide the number of channels
278         //                with an API instead of dynamically.
279 
280         // Find the max channels over all mixing lists.
281         const size_t num_mixed_channels = std::max(MaxNumChannels(&mixList),
282             std::max(MaxNumChannels(&additionalFramesList),
283                      MaxNumChannels(&rampOutList)));
284 
285         mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency,
286                                 AudioFrame::kNormalSpeech,
287                                 AudioFrame::kVadPassive, num_mixed_channels);
288 
289         _timeStamp += static_cast<uint32_t>(_sampleSize);
290 
291         // We only use the limiter if it supports the output sample rate and
292         // we're actually mixing multiple streams.
293         use_limiter_ =
294             _numMixedParticipants > 1 &&
295             _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz;
296 
297         MixFromList(mixedAudio, mixList);
298         MixAnonomouslyFromList(mixedAudio, additionalFramesList);
299         MixAnonomouslyFromList(mixedAudio, rampOutList);
300 
301         if(mixedAudio->samples_per_channel_ == 0) {
302             // Nothing was mixed, set the audio samples to silence.
303             mixedAudio->samples_per_channel_ = _sampleSize;
304             mixedAudio->Mute();
305         } else {
306             // Only call the limiter if we have something to mix.
307             if(!LimitMixedAudio(mixedAudio))
308                 retval = -1;
309         }
310     }
311 
312     {
313         CriticalSectionScoped cs(_cbCrit.get());
314         if(_mixReceiver != NULL) {
315             const AudioFrame** dummy = NULL;
316             _mixReceiver->NewMixedAudio(
317                 _id,
318                 *mixedAudio,
319                 dummy,
320                 0);
321         }
322     }
323 
324     // Reclaim all outstanding memory.
325     _audioFramePool->PushMemory(mixedAudio);
326     ClearAudioFrameList(&mixList);
327     ClearAudioFrameList(&rampOutList);
328     ClearAudioFrameList(&additionalFramesList);
329     {
330         CriticalSectionScoped cs(_crit.get());
331         _processCalls--;
332     }
333     return retval;
334 }
335 
RegisterMixedStreamCallback(AudioMixerOutputReceiver * mixReceiver)336 int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback(
337     AudioMixerOutputReceiver* mixReceiver) {
338     CriticalSectionScoped cs(_cbCrit.get());
339     if(_mixReceiver != NULL) {
340         return -1;
341     }
342     _mixReceiver = mixReceiver;
343     return 0;
344 }
345 
UnRegisterMixedStreamCallback()346 int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() {
347     CriticalSectionScoped cs(_cbCrit.get());
348     if(_mixReceiver == NULL) {
349         return -1;
350     }
351     _mixReceiver = NULL;
352     return 0;
353 }
354 
SetOutputFrequency(const Frequency & frequency)355 int32_t AudioConferenceMixerImpl::SetOutputFrequency(
356     const Frequency& frequency) {
357     CriticalSectionScoped cs(_crit.get());
358 
359     _outputFrequency = frequency;
360     _sampleSize =
361         static_cast<size_t>((_outputFrequency*kProcessPeriodicityInMs) / 1000);
362 
363     return 0;
364 }
365 
366 AudioConferenceMixer::Frequency
OutputFrequency() const367 AudioConferenceMixerImpl::OutputFrequency() const {
368     CriticalSectionScoped cs(_crit.get());
369     return _outputFrequency;
370 }
371 
SetMixabilityStatus(MixerParticipant * participant,bool mixable)372 int32_t AudioConferenceMixerImpl::SetMixabilityStatus(
373     MixerParticipant* participant, bool mixable) {
374     if (!mixable) {
375         // Anonymous participants are in a separate list. Make sure that the
376         // participant is in the _participantList if it is being mixed.
377         SetAnonymousMixabilityStatus(participant, false);
378     }
379     size_t numMixedParticipants;
380     {
381         CriticalSectionScoped cs(_cbCrit.get());
382         const bool isMixed =
383             IsParticipantInList(*participant, _participantList);
384         // API must be called with a new state.
385         if(!(mixable ^ isMixed)) {
386             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
387                          "Mixable is aready %s",
388                          isMixed ? "ON" : "off");
389             return -1;
390         }
391         bool success = false;
392         if(mixable) {
393             success = AddParticipantToList(participant, &_participantList);
394         } else {
395             success = RemoveParticipantFromList(participant, &_participantList);
396         }
397         if(!success) {
398             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
399                          "failed to %s participant",
400                          mixable ? "add" : "remove");
401             assert(false);
402             return -1;
403         }
404 
405         size_t numMixedNonAnonymous = _participantList.size();
406         if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) {
407             numMixedNonAnonymous = kMaximumAmountOfMixedParticipants;
408         }
409         numMixedParticipants =
410             numMixedNonAnonymous + _additionalParticipantList.size();
411     }
412     // A MixerParticipant was added or removed. Make sure the scratch
413     // buffer is updated if necessary.
414     // Note: The scratch buffer may only be updated in Process().
415     CriticalSectionScoped cs(_crit.get());
416     _numMixedParticipants = numMixedParticipants;
417     return 0;
418 }
419 
MixabilityStatus(const MixerParticipant & participant) const420 bool AudioConferenceMixerImpl::MixabilityStatus(
421     const MixerParticipant& participant) const {
422     CriticalSectionScoped cs(_cbCrit.get());
423     return IsParticipantInList(participant, _participantList);
424 }
425 
SetAnonymousMixabilityStatus(MixerParticipant * participant,bool anonymous)426 int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus(
427     MixerParticipant* participant, bool anonymous) {
428     CriticalSectionScoped cs(_cbCrit.get());
429     if(IsParticipantInList(*participant, _additionalParticipantList)) {
430         if(anonymous) {
431             return 0;
432         }
433         if(!RemoveParticipantFromList(participant,
434                                       &_additionalParticipantList)) {
435             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
436                          "unable to remove participant from anonymous list");
437             assert(false);
438             return -1;
439         }
440         return AddParticipantToList(participant, &_participantList) ? 0 : -1;
441     }
442     if(!anonymous) {
443         return 0;
444     }
445     const bool mixable = RemoveParticipantFromList(participant,
446                                                    &_participantList);
447     if(!mixable) {
448         WEBRTC_TRACE(
449             kTraceWarning,
450             kTraceAudioMixerServer,
451             _id,
452             "participant must be registered before turning it into anonymous");
453         // Setting anonymous status is only possible if MixerParticipant is
454         // already registered.
455         return -1;
456     }
457     return AddParticipantToList(participant, &_additionalParticipantList) ?
458         0 : -1;
459 }
460 
AnonymousMixabilityStatus(const MixerParticipant & participant) const461 bool AudioConferenceMixerImpl::AnonymousMixabilityStatus(
462     const MixerParticipant& participant) const {
463     CriticalSectionScoped cs(_cbCrit.get());
464     return IsParticipantInList(participant, _additionalParticipantList);
465 }
466 
SetMinimumMixingFrequency(Frequency freq)467 int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency(
468     Frequency freq) {
469     // Make sure that only allowed sampling frequencies are used. Use closest
470     // higher sampling frequency to avoid losing information.
471     if (static_cast<int>(freq) == 12000) {
472          freq = kWbInHz;
473     } else if (static_cast<int>(freq) == 24000) {
474         freq = kSwbInHz;
475     }
476 
477     if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) ||
478        (freq == kLowestPossible)) {
479         _minimumMixingFreq=freq;
480         return 0;
481     } else {
482         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
483                      "SetMinimumMixingFrequency incorrect frequency: %i",freq);
484         assert(false);
485         return -1;
486     }
487 }
488 
489 // Check all AudioFrames that are to be mixed. The highest sampling frequency
490 // found is the lowest that can be used without losing information.
GetLowestMixingFrequency() const491 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() const {
492     const int participantListFrequency =
493         GetLowestMixingFrequencyFromList(_participantList);
494     const int anonymousListFrequency =
495         GetLowestMixingFrequencyFromList(_additionalParticipantList);
496     const int highestFreq =
497         (participantListFrequency > anonymousListFrequency) ?
498             participantListFrequency : anonymousListFrequency;
499     // Check if the user specified a lowest mixing frequency.
500     if(_minimumMixingFreq != kLowestPossible) {
501         if(_minimumMixingFreq > highestFreq) {
502             return _minimumMixingFreq;
503         }
504     }
505     return highestFreq;
506 }
507 
GetLowestMixingFrequencyFromList(const MixerParticipantList & mixList) const508 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
509     const MixerParticipantList& mixList) const {
510     int32_t highestFreq = 8000;
511     for (MixerParticipantList::const_iterator iter = mixList.begin();
512          iter != mixList.end();
513          ++iter) {
514         const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
515         if(neededFrequency > highestFreq) {
516             highestFreq = neededFrequency;
517         }
518     }
519     return highestFreq;
520 }
521 
UpdateToMix(AudioFrameList * mixList,AudioFrameList * rampOutList,std::map<int,MixerParticipant * > * mixParticipantList,size_t * maxAudioFrameCounter) const522 void AudioConferenceMixerImpl::UpdateToMix(
523     AudioFrameList* mixList,
524     AudioFrameList* rampOutList,
525     std::map<int, MixerParticipant*>* mixParticipantList,
526     size_t* maxAudioFrameCounter) const {
527     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
528                  "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)",
529                  *maxAudioFrameCounter);
530     const size_t mixListStartSize = mixList->size();
531     AudioFrameList activeList;
532     // Struct needed by the passive lists to keep track of which AudioFrame
533     // belongs to which MixerParticipant.
534     ParticipantFramePairList passiveWasNotMixedList;
535     ParticipantFramePairList passiveWasMixedList;
536     for (MixerParticipantList::const_iterator participant =
537         _participantList.begin(); participant != _participantList.end();
538          ++participant) {
539         // Stop keeping track of passive participants if there are already
540         // enough participants available (they wont be mixed anyway).
541         bool mustAddToPassiveList = (*maxAudioFrameCounter >
542                                     (activeList.size() +
543                                      passiveWasMixedList.size() +
544                                      passiveWasNotMixedList.size()));
545 
546         bool wasMixed = false;
547         wasMixed = (*participant)->_mixHistory->WasMixed();
548         AudioFrame* audioFrame = NULL;
549         if(_audioFramePool->PopMemory(audioFrame) == -1) {
550             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
551                          "failed PopMemory() call");
552             assert(false);
553             return;
554         }
555         audioFrame->sample_rate_hz_ = _outputFrequency;
556 
557         if((*participant)->GetAudioFrame(_id, audioFrame) != 0) {
558             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
559                          "failed to GetAudioFrame() from participant");
560             _audioFramePool->PushMemory(audioFrame);
561             continue;
562         }
563         if (_participantList.size() != 1) {
564           // TODO(wu): Issue 3390, add support for multiple participants case.
565           audioFrame->ntp_time_ms_ = -1;
566         }
567 
568         // TODO(henrike): this assert triggers in some test cases where SRTP is
569         // used which prevents NetEQ from making a VAD. Temporarily disable this
570         // assert until the problem is fixed on a higher level.
571         // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown);
572         if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) {
573             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
574                          "invalid VAD state from participant");
575         }
576 
577         if(audioFrame->vad_activity_ == AudioFrame::kVadActive) {
578             if(!wasMixed) {
579                 RampIn(*audioFrame);
580             }
581 
582             if(activeList.size() >= *maxAudioFrameCounter) {
583                 // There are already more active participants than should be
584                 // mixed. Only keep the ones with the highest energy.
585                 AudioFrameList::iterator replaceItem;
586                 CalculateEnergy(*audioFrame);
587                 uint32_t lowestEnergy = audioFrame->energy_;
588 
589                 bool found_replace_item = false;
590                 for (AudioFrameList::iterator iter = activeList.begin();
591                      iter != activeList.end();
592                      ++iter) {
593                     CalculateEnergy(**iter);
594                     if((*iter)->energy_ < lowestEnergy) {
595                         replaceItem = iter;
596                         lowestEnergy = (*iter)->energy_;
597                         found_replace_item = true;
598                     }
599                 }
600                 if(found_replace_item) {
601                     AudioFrame* replaceFrame = *replaceItem;
602 
603                     bool replaceWasMixed = false;
604                     std::map<int, MixerParticipant*>::const_iterator it =
605                         mixParticipantList->find(replaceFrame->id_);
606 
607                     // When a frame is pushed to |activeList| it is also pushed
608                     // to mixParticipantList with the frame's id. This means
609                     // that the Find call above should never fail.
610                     assert(it != mixParticipantList->end());
611                     replaceWasMixed = it->second->_mixHistory->WasMixed();
612 
613                     mixParticipantList->erase(replaceFrame->id_);
614                     activeList.erase(replaceItem);
615 
616                     activeList.push_front(audioFrame);
617                     (*mixParticipantList)[audioFrame->id_] = *participant;
618                     assert(mixParticipantList->size() <=
619                            kMaximumAmountOfMixedParticipants);
620 
621                     if (replaceWasMixed) {
622                       RampOut(*replaceFrame);
623                       rampOutList->push_back(replaceFrame);
624                       assert(rampOutList->size() <=
625                              kMaximumAmountOfMixedParticipants);
626                     } else {
627                       _audioFramePool->PushMemory(replaceFrame);
628                     }
629                 } else {
630                     if(wasMixed) {
631                         RampOut(*audioFrame);
632                         rampOutList->push_back(audioFrame);
633                         assert(rampOutList->size() <=
634                                kMaximumAmountOfMixedParticipants);
635                     } else {
636                         _audioFramePool->PushMemory(audioFrame);
637                     }
638                 }
639             } else {
640                 activeList.push_front(audioFrame);
641                 (*mixParticipantList)[audioFrame->id_] = *participant;
642                 assert(mixParticipantList->size() <=
643                        kMaximumAmountOfMixedParticipants);
644             }
645         } else {
646             if(wasMixed) {
647                 ParticipantFramePair* pair = new ParticipantFramePair;
648                 pair->audioFrame  = audioFrame;
649                 pair->participant = *participant;
650                 passiveWasMixedList.push_back(pair);
651             } else if(mustAddToPassiveList) {
652                 RampIn(*audioFrame);
653                 ParticipantFramePair* pair = new ParticipantFramePair;
654                 pair->audioFrame  = audioFrame;
655                 pair->participant = *participant;
656                 passiveWasNotMixedList.push_back(pair);
657             } else {
658                 _audioFramePool->PushMemory(audioFrame);
659             }
660         }
661     }
662     assert(activeList.size() <= *maxAudioFrameCounter);
663     // At this point it is known which participants should be mixed. Transfer
664     // this information to this functions output parameters.
665     for (AudioFrameList::const_iterator iter = activeList.begin();
666          iter != activeList.end();
667          ++iter) {
668         mixList->push_back(*iter);
669     }
670     activeList.clear();
671     // Always mix a constant number of AudioFrames. If there aren't enough
672     // active participants mix passive ones. Starting with those that was mixed
673     // last iteration.
674     for (ParticipantFramePairList::const_iterator
675         iter = passiveWasMixedList.begin(); iter != passiveWasMixedList.end();
676          ++iter) {
677         if(mixList->size() < *maxAudioFrameCounter + mixListStartSize) {
678             mixList->push_back((*iter)->audioFrame);
679             (*mixParticipantList)[(*iter)->audioFrame->id_] =
680                 (*iter)->participant;
681             assert(mixParticipantList->size() <=
682                    kMaximumAmountOfMixedParticipants);
683         } else {
684             _audioFramePool->PushMemory((*iter)->audioFrame);
685         }
686         delete *iter;
687     }
688     // And finally the ones that have not been mixed for a while.
689     for (ParticipantFramePairList::const_iterator iter =
690              passiveWasNotMixedList.begin();
691          iter != passiveWasNotMixedList.end();
692          ++iter) {
693         if(mixList->size() <  *maxAudioFrameCounter + mixListStartSize) {
694           mixList->push_back((*iter)->audioFrame);
695             (*mixParticipantList)[(*iter)->audioFrame->id_] =
696                 (*iter)->participant;
697             assert(mixParticipantList->size() <=
698                    kMaximumAmountOfMixedParticipants);
699         } else {
700             _audioFramePool->PushMemory((*iter)->audioFrame);
701         }
702         delete *iter;
703     }
704     assert(*maxAudioFrameCounter + mixListStartSize >= mixList->size());
705     *maxAudioFrameCounter += mixListStartSize - mixList->size();
706 }
707 
GetAdditionalAudio(AudioFrameList * additionalFramesList) const708 void AudioConferenceMixerImpl::GetAdditionalAudio(
709     AudioFrameList* additionalFramesList) const {
710     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
711                  "GetAdditionalAudio(additionalFramesList)");
712     // The GetAudioFrame() callback may result in the participant being removed
713     // from additionalParticipantList_. If that happens it will invalidate any
714     // iterators. Create a copy of the participants list such that the list of
715     // participants can be traversed safely.
716     MixerParticipantList additionalParticipantList;
717     additionalParticipantList.insert(additionalParticipantList.begin(),
718                                      _additionalParticipantList.begin(),
719                                      _additionalParticipantList.end());
720 
721     for (MixerParticipantList::const_iterator participant =
722              additionalParticipantList.begin();
723          participant != additionalParticipantList.end();
724          ++participant) {
725         AudioFrame* audioFrame = NULL;
726         if(_audioFramePool->PopMemory(audioFrame) == -1) {
727             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
728                          "failed PopMemory() call");
729             assert(false);
730             return;
731         }
732         audioFrame->sample_rate_hz_ = _outputFrequency;
733         if((*participant)->GetAudioFrame(_id, audioFrame) != 0) {
734             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
735                          "failed to GetAudioFrame() from participant");
736             _audioFramePool->PushMemory(audioFrame);
737             continue;
738         }
739         if(audioFrame->samples_per_channel_ == 0) {
740             // Empty frame. Don't use it.
741             _audioFramePool->PushMemory(audioFrame);
742             continue;
743         }
744         additionalFramesList->push_back(audioFrame);
745     }
746 }
747 
UpdateMixedStatus(const std::map<int,MixerParticipant * > & mixedParticipantsMap) const748 void AudioConferenceMixerImpl::UpdateMixedStatus(
749     const std::map<int, MixerParticipant*>& mixedParticipantsMap) const {
750     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
751                  "UpdateMixedStatus(mixedParticipantsMap)");
752     assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants);
753 
754     // Loop through all participants. If they are in the mix map they
755     // were mixed.
756     for (MixerParticipantList::const_iterator
757         participant =_participantList.begin();
758         participant != _participantList.end();
759          ++participant) {
760         bool isMixed = false;
761         for (std::map<int, MixerParticipant*>::const_iterator it =
762                  mixedParticipantsMap.begin();
763              it != mixedParticipantsMap.end();
764              ++it) {
765           if (it->second == *participant) {
766             isMixed = true;
767             break;
768           }
769         }
770         (*participant)->_mixHistory->SetIsMixed(isMixed);
771     }
772 }
773 
ClearAudioFrameList(AudioFrameList * audioFrameList) const774 void AudioConferenceMixerImpl::ClearAudioFrameList(
775     AudioFrameList* audioFrameList) const {
776     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
777                  "ClearAudioFrameList(audioFrameList)");
778     for (AudioFrameList::iterator iter = audioFrameList->begin();
779          iter != audioFrameList->end();
780          ++iter) {
781         _audioFramePool->PushMemory(*iter);
782     }
783     audioFrameList->clear();
784 }
785 
UpdateVADPositiveParticipants(AudioFrameList * mixList) const786 void AudioConferenceMixerImpl::UpdateVADPositiveParticipants(
787     AudioFrameList* mixList) const {
788     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
789                  "UpdateVADPositiveParticipants(mixList)");
790 
791     for (AudioFrameList::const_iterator iter = mixList->begin();
792          iter != mixList->end();
793          ++iter) {
794         CalculateEnergy(**iter);
795     }
796 }
797 
IsParticipantInList(const MixerParticipant & participant,const MixerParticipantList & participantList) const798 bool AudioConferenceMixerImpl::IsParticipantInList(
799     const MixerParticipant& participant,
800     const MixerParticipantList& participantList) const {
801     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
802                  "IsParticipantInList(participant,participantList)");
803     for (MixerParticipantList::const_iterator iter = participantList.begin();
804          iter != participantList.end();
805          ++iter) {
806         if(&participant == *iter) {
807             return true;
808         }
809     }
810     return false;
811 }
812 
AddParticipantToList(MixerParticipant * participant,MixerParticipantList * participantList) const813 bool AudioConferenceMixerImpl::AddParticipantToList(
814     MixerParticipant* participant,
815     MixerParticipantList* participantList) const {
816     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
817                  "AddParticipantToList(participant, participantList)");
818     participantList->push_back(participant);
819     // Make sure that the mixed status is correct for new MixerParticipant.
820     participant->_mixHistory->ResetMixedStatus();
821     return true;
822 }
823 
RemoveParticipantFromList(MixerParticipant * participant,MixerParticipantList * participantList) const824 bool AudioConferenceMixerImpl::RemoveParticipantFromList(
825     MixerParticipant* participant,
826     MixerParticipantList* participantList) const {
827     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
828                  "RemoveParticipantFromList(participant, participantList)");
829     for (MixerParticipantList::iterator iter = participantList->begin();
830          iter != participantList->end();
831          ++iter) {
832         if(*iter == participant) {
833             participantList->erase(iter);
834             // Participant is no longer mixed, reset to default.
835             participant->_mixHistory->ResetMixedStatus();
836             return true;
837         }
838     }
839     return false;
840 }
841 
MixFromList(AudioFrame * mixedAudio,const AudioFrameList & audioFrameList) const842 int32_t AudioConferenceMixerImpl::MixFromList(
843     AudioFrame* mixedAudio,
844     const AudioFrameList& audioFrameList) const {
845     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
846                  "MixFromList(mixedAudio, audioFrameList)");
847     if(audioFrameList.empty()) return 0;
848 
849     uint32_t position = 0;
850 
851     if (_numMixedParticipants == 1) {
852       mixedAudio->timestamp_ = audioFrameList.front()->timestamp_;
853       mixedAudio->elapsed_time_ms_ = audioFrameList.front()->elapsed_time_ms_;
854     } else {
855       // TODO(wu): Issue 3390.
856       // Audio frame timestamp is only supported in one channel case.
857       mixedAudio->timestamp_ = 0;
858       mixedAudio->elapsed_time_ms_ = -1;
859     }
860 
861     for (AudioFrameList::const_iterator iter = audioFrameList.begin();
862          iter != audioFrameList.end();
863          ++iter) {
864         if(position >= kMaximumAmountOfMixedParticipants) {
865             WEBRTC_TRACE(
866                 kTraceMemory,
867                 kTraceAudioMixerServer,
868                 _id,
869                 "Trying to mix more than max amount of mixed participants:%d!",
870                 kMaximumAmountOfMixedParticipants);
871             // Assert and avoid crash
872             assert(false);
873             position = 0;
874         }
875         MixFrames(mixedAudio, (*iter), use_limiter_);
876 
877         position++;
878     }
879 
880     return 0;
881 }
882 
883 // TODO(andrew): consolidate this function with MixFromList.
MixAnonomouslyFromList(AudioFrame * mixedAudio,const AudioFrameList & audioFrameList) const884 int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList(
885     AudioFrame* mixedAudio,
886     const AudioFrameList& audioFrameList) const {
887     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
888                  "MixAnonomouslyFromList(mixedAudio, audioFrameList)");
889 
890     if(audioFrameList.empty()) return 0;
891 
892     for (AudioFrameList::const_iterator iter = audioFrameList.begin();
893          iter != audioFrameList.end();
894          ++iter) {
895         MixFrames(mixedAudio, *iter, use_limiter_);
896     }
897     return 0;
898 }
899 
LimitMixedAudio(AudioFrame * mixedAudio) const900 bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame* mixedAudio) const {
901     if (!use_limiter_) {
902       return true;
903     }
904 
905     // Smoothly limit the mixed frame.
906     const int error = _limiter->ProcessStream(mixedAudio);
907 
908     // And now we can safely restore the level. This procedure results in
909     // some loss of resolution, deemed acceptable.
910     //
911     // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
912     // and compression gain of 6 dB). However, in the transition frame when this
913     // is enabled (moving from one to two participants) it has the potential to
914     // create discontinuities in the mixed frame.
915     //
916     // Instead we double the frame (with addition since left-shifting a
917     // negative value is undefined).
918     *mixedAudio += *mixedAudio;
919 
920     if(error != _limiter->kNoError) {
921         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
922                      "Error from AudioProcessing: %d", error);
923         assert(false);
924         return false;
925     }
926     return true;
927 }
928 }  // namespace webrtc
929