• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/audio_coding/neteq/neteq_impl.h"
12 
13 #include <assert.h>
14 #include <memory.h>  // memset
15 
16 #include <algorithm>
17 
18 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
19 #include "webrtc/modules/audio_coding/neteq/accelerate.h"
20 #include "webrtc/modules/audio_coding/neteq/background_noise.h"
21 #include "webrtc/modules/audio_coding/neteq/buffer_level_filter.h"
22 #include "webrtc/modules/audio_coding/neteq/comfort_noise.h"
23 #include "webrtc/modules/audio_coding/neteq/decision_logic.h"
24 #include "webrtc/modules/audio_coding/neteq/decoder_database.h"
25 #include "webrtc/modules/audio_coding/neteq/defines.h"
26 #include "webrtc/modules/audio_coding/neteq/delay_manager.h"
27 #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h"
28 #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h"
29 #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h"
30 #include "webrtc/modules/audio_coding/neteq/expand.h"
31 #include "webrtc/modules/audio_coding/neteq/interface/audio_decoder.h"
32 #include "webrtc/modules/audio_coding/neteq/merge.h"
33 #include "webrtc/modules/audio_coding/neteq/normal.h"
34 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
35 #include "webrtc/modules/audio_coding/neteq/packet.h"
36 #include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
37 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
38 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
39 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
40 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
41 #include "webrtc/modules/interface/module_common_types.h"
42 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
43 #include "webrtc/system_wrappers/interface/logging.h"
44 
45 // Modify the code to obtain backwards bit-exactness. Once bit-exactness is no
46 // longer required, this #define should be removed (and the code that it
47 // enables).
48 #define LEGACY_BITEXACT
49 
50 namespace webrtc {
51 
NetEqImpl(int fs,BufferLevelFilter * buffer_level_filter,DecoderDatabase * decoder_database,DelayManager * delay_manager,DelayPeakDetector * delay_peak_detector,DtmfBuffer * dtmf_buffer,DtmfToneGenerator * dtmf_tone_generator,PacketBuffer * packet_buffer,PayloadSplitter * payload_splitter,TimestampScaler * timestamp_scaler,AccelerateFactory * accelerate_factory,ExpandFactory * expand_factory,PreemptiveExpandFactory * preemptive_expand_factory,bool create_components)52 NetEqImpl::NetEqImpl(int fs,
53                      BufferLevelFilter* buffer_level_filter,
54                      DecoderDatabase* decoder_database,
55                      DelayManager* delay_manager,
56                      DelayPeakDetector* delay_peak_detector,
57                      DtmfBuffer* dtmf_buffer,
58                      DtmfToneGenerator* dtmf_tone_generator,
59                      PacketBuffer* packet_buffer,
60                      PayloadSplitter* payload_splitter,
61                      TimestampScaler* timestamp_scaler,
62                      AccelerateFactory* accelerate_factory,
63                      ExpandFactory* expand_factory,
64                      PreemptiveExpandFactory* preemptive_expand_factory,
65                      bool create_components)
66     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
67       buffer_level_filter_(buffer_level_filter),
68       decoder_database_(decoder_database),
69       delay_manager_(delay_manager),
70       delay_peak_detector_(delay_peak_detector),
71       dtmf_buffer_(dtmf_buffer),
72       dtmf_tone_generator_(dtmf_tone_generator),
73       packet_buffer_(packet_buffer),
74       payload_splitter_(payload_splitter),
75       timestamp_scaler_(timestamp_scaler),
76       vad_(new PostDecodeVad()),
77       expand_factory_(expand_factory),
78       accelerate_factory_(accelerate_factory),
79       preemptive_expand_factory_(preemptive_expand_factory),
80       last_mode_(kModeNormal),
81       decoded_buffer_length_(kMaxFrameSize),
82       decoded_buffer_(new int16_t[decoded_buffer_length_]),
83       playout_timestamp_(0),
84       new_codec_(false),
85       timestamp_(0),
86       reset_decoder_(false),
87       current_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
88       current_cng_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
89       ssrc_(0),
90       first_packet_(true),
91       error_code_(0),
92       decoder_error_code_(0),
93       decoded_packet_sequence_number_(-1),
94       decoded_packet_timestamp_(0) {
95   if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) {
96     LOG(LS_ERROR) << "Sample rate " << fs << " Hz not supported. " <<
97         "Changing to 8000 Hz.";
98     fs = 8000;
99   }
100   LOG(LS_VERBOSE) << "Create NetEqImpl object with fs = " << fs << ".";
101   fs_hz_ = fs;
102   fs_mult_ = fs / 8000;
103   output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
104   decoder_frame_length_ = 3 * output_size_samples_;
105   WebRtcSpl_Init();
106   if (create_components) {
107     SetSampleRateAndChannels(fs, 1);  // Default is 1 channel.
108   }
109 }
110 
~NetEqImpl()111 NetEqImpl::~NetEqImpl() {
112   LOG(LS_INFO) << "Deleting NetEqImpl object.";
113 }
114 
InsertPacket(const WebRtcRTPHeader & rtp_header,const uint8_t * payload,int length_bytes,uint32_t receive_timestamp)115 int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header,
116                             const uint8_t* payload,
117                             int length_bytes,
118                             uint32_t receive_timestamp) {
119   CriticalSectionScoped lock(crit_sect_.get());
120   LOG(LS_VERBOSE) << "InsertPacket: ts=" << rtp_header.header.timestamp <<
121       ", sn=" << rtp_header.header.sequenceNumber <<
122       ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
123       ", ssrc=" << rtp_header.header.ssrc <<
124       ", len=" << length_bytes;
125   int error = InsertPacketInternal(rtp_header, payload, length_bytes,
126                                    receive_timestamp, false);
127   if (error != 0) {
128     LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
129     error_code_ = error;
130     return kFail;
131   }
132   return kOK;
133 }
134 
InsertSyncPacket(const WebRtcRTPHeader & rtp_header,uint32_t receive_timestamp)135 int NetEqImpl::InsertSyncPacket(const WebRtcRTPHeader& rtp_header,
136                                 uint32_t receive_timestamp) {
137   CriticalSectionScoped lock(crit_sect_.get());
138   LOG(LS_VERBOSE) << "InsertPacket-Sync: ts="
139       << rtp_header.header.timestamp <<
140       ", sn=" << rtp_header.header.sequenceNumber <<
141       ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
142       ", ssrc=" << rtp_header.header.ssrc;
143 
144   const uint8_t kSyncPayload[] = { 's', 'y', 'n', 'c' };
145   int error = InsertPacketInternal(
146       rtp_header, kSyncPayload, sizeof(kSyncPayload), receive_timestamp, true);
147 
148   if (error != 0) {
149     LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
150     error_code_ = error;
151     return kFail;
152   }
153   return kOK;
154 }
155 
GetAudio(size_t max_length,int16_t * output_audio,int * samples_per_channel,int * num_channels,NetEqOutputType * type)156 int NetEqImpl::GetAudio(size_t max_length, int16_t* output_audio,
157                         int* samples_per_channel, int* num_channels,
158                         NetEqOutputType* type) {
159   CriticalSectionScoped lock(crit_sect_.get());
160   LOG(LS_VERBOSE) << "GetAudio";
161   int error = GetAudioInternal(max_length, output_audio, samples_per_channel,
162                                num_channels);
163   LOG(LS_VERBOSE) << "Produced " << *samples_per_channel <<
164       " samples/channel for " << *num_channels << " channel(s)";
165   if (error != 0) {
166     LOG_FERR1(LS_WARNING, GetAudioInternal, error);
167     error_code_ = error;
168     return kFail;
169   }
170   if (type) {
171     *type = LastOutputType();
172   }
173   return kOK;
174 }
175 
RegisterPayloadType(enum NetEqDecoder codec,uint8_t rtp_payload_type)176 int NetEqImpl::RegisterPayloadType(enum NetEqDecoder codec,
177                                    uint8_t rtp_payload_type) {
178   CriticalSectionScoped lock(crit_sect_.get());
179   LOG_API2(static_cast<int>(rtp_payload_type), codec);
180   int ret = decoder_database_->RegisterPayload(rtp_payload_type, codec);
181   if (ret != DecoderDatabase::kOK) {
182     LOG_FERR2(LS_WARNING, RegisterPayload, rtp_payload_type, codec);
183     switch (ret) {
184       case DecoderDatabase::kInvalidRtpPayloadType:
185         error_code_ = kInvalidRtpPayloadType;
186         break;
187       case DecoderDatabase::kCodecNotSupported:
188         error_code_ = kCodecNotSupported;
189         break;
190       case DecoderDatabase::kDecoderExists:
191         error_code_ = kDecoderExists;
192         break;
193       default:
194         error_code_ = kOtherError;
195     }
196     return kFail;
197   }
198   return kOK;
199 }
200 
RegisterExternalDecoder(AudioDecoder * decoder,enum NetEqDecoder codec,uint8_t rtp_payload_type)201 int NetEqImpl::RegisterExternalDecoder(AudioDecoder* decoder,
202                                        enum NetEqDecoder codec,
203                                        uint8_t rtp_payload_type) {
204   CriticalSectionScoped lock(crit_sect_.get());
205   LOG_API2(static_cast<int>(rtp_payload_type), codec);
206   if (!decoder) {
207     LOG(LS_ERROR) << "Cannot register external decoder with NULL pointer";
208     assert(false);
209     return kFail;
210   }
211   const int sample_rate_hz = AudioDecoder::CodecSampleRateHz(codec);
212   int ret = decoder_database_->InsertExternal(rtp_payload_type, codec,
213                                               sample_rate_hz, decoder);
214   if (ret != DecoderDatabase::kOK) {
215     LOG_FERR2(LS_WARNING, InsertExternal, rtp_payload_type, codec);
216     switch (ret) {
217       case DecoderDatabase::kInvalidRtpPayloadType:
218         error_code_ = kInvalidRtpPayloadType;
219         break;
220       case DecoderDatabase::kCodecNotSupported:
221         error_code_ = kCodecNotSupported;
222         break;
223       case DecoderDatabase::kDecoderExists:
224         error_code_ = kDecoderExists;
225         break;
226       case DecoderDatabase::kInvalidSampleRate:
227         error_code_ = kInvalidSampleRate;
228         break;
229       case DecoderDatabase::kInvalidPointer:
230         error_code_ = kInvalidPointer;
231         break;
232       default:
233         error_code_ = kOtherError;
234     }
235     return kFail;
236   }
237   return kOK;
238 }
239 
RemovePayloadType(uint8_t rtp_payload_type)240 int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) {
241   CriticalSectionScoped lock(crit_sect_.get());
242   LOG_API1(static_cast<int>(rtp_payload_type));
243   int ret = decoder_database_->Remove(rtp_payload_type);
244   if (ret == DecoderDatabase::kOK) {
245     return kOK;
246   } else if (ret == DecoderDatabase::kDecoderNotFound) {
247     error_code_ = kDecoderNotFound;
248   } else {
249     error_code_ = kOtherError;
250   }
251   LOG_FERR1(LS_WARNING, Remove, rtp_payload_type);
252   return kFail;
253 }
254 
SetMinimumDelay(int delay_ms)255 bool NetEqImpl::SetMinimumDelay(int delay_ms) {
256   CriticalSectionScoped lock(crit_sect_.get());
257   if (delay_ms >= 0 && delay_ms < 10000) {
258     assert(delay_manager_.get());
259     return delay_manager_->SetMinimumDelay(delay_ms);
260   }
261   return false;
262 }
263 
SetMaximumDelay(int delay_ms)264 bool NetEqImpl::SetMaximumDelay(int delay_ms) {
265   CriticalSectionScoped lock(crit_sect_.get());
266   if (delay_ms >= 0 && delay_ms < 10000) {
267     assert(delay_manager_.get());
268     return delay_manager_->SetMaximumDelay(delay_ms);
269   }
270   return false;
271 }
272 
LeastRequiredDelayMs() const273 int NetEqImpl::LeastRequiredDelayMs() const {
274   CriticalSectionScoped lock(crit_sect_.get());
275   assert(delay_manager_.get());
276   return delay_manager_->least_required_delay_ms();
277 }
278 
SetPlayoutMode(NetEqPlayoutMode mode)279 void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) {
280   CriticalSectionScoped lock(crit_sect_.get());
281   if (!decision_logic_.get() || mode != decision_logic_->playout_mode()) {
282     // The reset() method calls delete for the old object.
283     CreateDecisionLogic(mode);
284   }
285 }
286 
PlayoutMode() const287 NetEqPlayoutMode NetEqImpl::PlayoutMode() const {
288   CriticalSectionScoped lock(crit_sect_.get());
289   assert(decision_logic_.get());
290   return decision_logic_->playout_mode();
291 }
292 
NetworkStatistics(NetEqNetworkStatistics * stats)293 int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) {
294   CriticalSectionScoped lock(crit_sect_.get());
295   assert(decoder_database_.get());
296   const int total_samples_in_buffers = packet_buffer_->NumSamplesInBuffer(
297       decoder_database_.get(), decoder_frame_length_) +
298           static_cast<int>(sync_buffer_->FutureLength());
299   assert(delay_manager_.get());
300   assert(decision_logic_.get());
301   stats_.GetNetworkStatistics(fs_hz_, total_samples_in_buffers,
302                               decoder_frame_length_, *delay_manager_.get(),
303                               *decision_logic_.get(), stats);
304   return 0;
305 }
306 
WaitingTimes(std::vector<int> * waiting_times)307 void NetEqImpl::WaitingTimes(std::vector<int>* waiting_times) {
308   CriticalSectionScoped lock(crit_sect_.get());
309   stats_.WaitingTimes(waiting_times);
310 }
311 
GetRtcpStatistics(RtcpStatistics * stats)312 void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) {
313   CriticalSectionScoped lock(crit_sect_.get());
314   if (stats) {
315     rtcp_.GetStatistics(false, stats);
316   }
317 }
318 
GetRtcpStatisticsNoReset(RtcpStatistics * stats)319 void NetEqImpl::GetRtcpStatisticsNoReset(RtcpStatistics* stats) {
320   CriticalSectionScoped lock(crit_sect_.get());
321   if (stats) {
322     rtcp_.GetStatistics(true, stats);
323   }
324 }
325 
EnableVad()326 void NetEqImpl::EnableVad() {
327   CriticalSectionScoped lock(crit_sect_.get());
328   assert(vad_.get());
329   vad_->Enable();
330 }
331 
DisableVad()332 void NetEqImpl::DisableVad() {
333   CriticalSectionScoped lock(crit_sect_.get());
334   assert(vad_.get());
335   vad_->Disable();
336 }
337 
GetPlayoutTimestamp(uint32_t * timestamp)338 bool NetEqImpl::GetPlayoutTimestamp(uint32_t* timestamp) {
339   CriticalSectionScoped lock(crit_sect_.get());
340   if (first_packet_) {
341     // We don't have a valid RTP timestamp until we have decoded our first
342     // RTP packet.
343     return false;
344   }
345   *timestamp = timestamp_scaler_->ToExternal(playout_timestamp_);
346   return true;
347 }
348 
LastError()349 int NetEqImpl::LastError() {
350   CriticalSectionScoped lock(crit_sect_.get());
351   return error_code_;
352 }
353 
LastDecoderError()354 int NetEqImpl::LastDecoderError() {
355   CriticalSectionScoped lock(crit_sect_.get());
356   return decoder_error_code_;
357 }
358 
FlushBuffers()359 void NetEqImpl::FlushBuffers() {
360   CriticalSectionScoped lock(crit_sect_.get());
361   LOG_API0();
362   packet_buffer_->Flush();
363   assert(sync_buffer_.get());
364   assert(expand_.get());
365   sync_buffer_->Flush();
366   sync_buffer_->set_next_index(sync_buffer_->next_index() -
367                                expand_->overlap_length());
368   // Set to wait for new codec.
369   first_packet_ = true;
370 }
371 
PacketBufferStatistics(int * current_num_packets,int * max_num_packets) const372 void NetEqImpl::PacketBufferStatistics(int* current_num_packets,
373                                        int* max_num_packets) const {
374   CriticalSectionScoped lock(crit_sect_.get());
375   packet_buffer_->BufferStat(current_num_packets, max_num_packets);
376 }
377 
DecodedRtpInfo(int * sequence_number,uint32_t * timestamp) const378 int NetEqImpl::DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const {
379   CriticalSectionScoped lock(crit_sect_.get());
380   if (decoded_packet_sequence_number_ < 0)
381     return -1;
382   *sequence_number = decoded_packet_sequence_number_;
383   *timestamp = decoded_packet_timestamp_;
384   return 0;
385 }
386 
SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode)387 void NetEqImpl::SetBackgroundNoiseMode(NetEqBackgroundNoiseMode mode) {
388   CriticalSectionScoped lock(crit_sect_.get());
389   assert(background_noise_.get());
390   background_noise_->set_mode(mode);
391 }
392 
BackgroundNoiseMode() const393 NetEqBackgroundNoiseMode NetEqImpl::BackgroundNoiseMode() const {
394   CriticalSectionScoped lock(crit_sect_.get());
395   assert(background_noise_.get());
396   return background_noise_->mode();
397 }
398 
sync_buffer_for_test() const399 const SyncBuffer* NetEqImpl::sync_buffer_for_test() const {
400   CriticalSectionScoped lock(crit_sect_.get());
401   return sync_buffer_.get();
402 }
403 
404 // Methods below this line are private.
405 
InsertPacketInternal(const WebRtcRTPHeader & rtp_header,const uint8_t * payload,int length_bytes,uint32_t receive_timestamp,bool is_sync_packet)406 int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
407                                     const uint8_t* payload,
408                                     int length_bytes,
409                                     uint32_t receive_timestamp,
410                                     bool is_sync_packet) {
411   if (!payload) {
412     LOG_F(LS_ERROR) << "payload == NULL";
413     return kInvalidPointer;
414   }
415   // Sanity checks for sync-packets.
416   if (is_sync_packet) {
417     if (decoder_database_->IsDtmf(rtp_header.header.payloadType) ||
418         decoder_database_->IsRed(rtp_header.header.payloadType) ||
419         decoder_database_->IsComfortNoise(rtp_header.header.payloadType)) {
420       LOG_F(LS_ERROR) << "Sync-packet with an unacceptable payload type "
421           << rtp_header.header.payloadType;
422       return kSyncPacketNotAccepted;
423     }
424     if (first_packet_ ||
425         rtp_header.header.payloadType != current_rtp_payload_type_ ||
426         rtp_header.header.ssrc != ssrc_) {
427       // Even if |current_rtp_payload_type_| is 0xFF, sync-packet isn't
428       // accepted.
429       LOG_F(LS_ERROR) << "Changing codec, SSRC or first packet "
430           "with sync-packet.";
431       return kSyncPacketNotAccepted;
432     }
433   }
434   PacketList packet_list;
435   RTPHeader main_header;
436   {
437     // Convert to Packet.
438     // Create |packet| within this separate scope, since it should not be used
439     // directly once it's been inserted in the packet list. This way, |packet|
440     // is not defined outside of this block.
441     Packet* packet = new Packet;
442     packet->header.markerBit = false;
443     packet->header.payloadType = rtp_header.header.payloadType;
444     packet->header.sequenceNumber = rtp_header.header.sequenceNumber;
445     packet->header.timestamp = rtp_header.header.timestamp;
446     packet->header.ssrc = rtp_header.header.ssrc;
447     packet->header.numCSRCs = 0;
448     packet->payload_length = length_bytes;
449     packet->primary = true;
450     packet->waiting_time = 0;
451     packet->payload = new uint8_t[packet->payload_length];
452     packet->sync_packet = is_sync_packet;
453     if (!packet->payload) {
454       LOG_F(LS_ERROR) << "Payload pointer is NULL.";
455     }
456     assert(payload);  // Already checked above.
457     memcpy(packet->payload, payload, packet->payload_length);
458     // Insert packet in a packet list.
459     packet_list.push_back(packet);
460     // Save main payloads header for later.
461     memcpy(&main_header, &packet->header, sizeof(main_header));
462   }
463 
464   bool update_sample_rate_and_channels = false;
465   // Reinitialize NetEq if it's needed (changed SSRC or first call).
466   if ((main_header.ssrc != ssrc_) || first_packet_) {
467     rtcp_.Init(main_header.sequenceNumber);
468     first_packet_ = false;
469 
470     // Flush the packet buffer and DTMF buffer.
471     packet_buffer_->Flush();
472     dtmf_buffer_->Flush();
473 
474     // Store new SSRC.
475     ssrc_ = main_header.ssrc;
476 
477     // Update audio buffer timestamp.
478     sync_buffer_->IncreaseEndTimestamp(main_header.timestamp - timestamp_);
479 
480     // Update codecs.
481     timestamp_ = main_header.timestamp;
482     current_rtp_payload_type_ = main_header.payloadType;
483 
484     // Set MCU to update codec on next SignalMCU call.
485     new_codec_ = true;
486 
487     // Reset timestamp scaling.
488     timestamp_scaler_->Reset();
489 
490     // Triger an update of sampling rate and the number of channels.
491     update_sample_rate_and_channels = true;
492   }
493 
494   // Update RTCP statistics, only for regular packets.
495   if (!is_sync_packet)
496     rtcp_.Update(main_header, receive_timestamp);
497 
498   // Check for RED payload type, and separate payloads into several packets.
499   if (decoder_database_->IsRed(main_header.payloadType)) {
500     assert(!is_sync_packet);  // We had a sanity check for this.
501     if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
502       LOG_FERR1(LS_WARNING, SplitRed, packet_list.size());
503       PacketBuffer::DeleteAllPackets(&packet_list);
504       return kRedundancySplitError;
505     }
506     // Only accept a few RED payloads of the same type as the main data,
507     // DTMF events and CNG.
508     payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
509     // Update the stored main payload header since the main payload has now
510     // changed.
511     memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
512   }
513 
514   // Check payload types.
515   if (decoder_database_->CheckPayloadTypes(packet_list) ==
516       DecoderDatabase::kDecoderNotFound) {
517     LOG_FERR1(LS_WARNING, CheckPayloadTypes, packet_list.size());
518     PacketBuffer::DeleteAllPackets(&packet_list);
519     return kUnknownRtpPayloadType;
520   }
521 
522   // Scale timestamp to internal domain (only for some codecs).
523   timestamp_scaler_->ToInternal(&packet_list);
524 
525   // Process DTMF payloads. Cycle through the list of packets, and pick out any
526   // DTMF payloads found.
527   PacketList::iterator it = packet_list.begin();
528   while (it != packet_list.end()) {
529     Packet* current_packet = (*it);
530     assert(current_packet);
531     assert(current_packet->payload);
532     if (decoder_database_->IsDtmf(current_packet->header.payloadType)) {
533       assert(!current_packet->sync_packet);  // We had a sanity check for this.
534       DtmfEvent event;
535       int ret = DtmfBuffer::ParseEvent(
536           current_packet->header.timestamp,
537           current_packet->payload,
538           current_packet->payload_length,
539           &event);
540       if (ret != DtmfBuffer::kOK) {
541         LOG_FERR2(LS_WARNING, ParseEvent, ret,
542                   current_packet->payload_length);
543         PacketBuffer::DeleteAllPackets(&packet_list);
544         return kDtmfParsingError;
545       }
546       if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) {
547         LOG_FERR0(LS_WARNING, InsertEvent);
548         PacketBuffer::DeleteAllPackets(&packet_list);
549         return kDtmfInsertError;
550       }
551       // TODO(hlundin): Let the destructor of Packet handle the payload.
552       delete [] current_packet->payload;
553       delete current_packet;
554       it = packet_list.erase(it);
555     } else {
556       ++it;
557     }
558   }
559 
560   // Check for FEC in packets, and separate payloads into several packets.
561   int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get());
562   if (ret != PayloadSplitter::kOK) {
563     LOG_FERR1(LS_WARNING, SplitFec, packet_list.size());
564     PacketBuffer::DeleteAllPackets(&packet_list);
565     switch (ret) {
566       case PayloadSplitter::kUnknownPayloadType:
567         return kUnknownRtpPayloadType;
568       default:
569         return kOtherError;
570     }
571   }
572 
573   // Split payloads into smaller chunks. This also verifies that all payloads
574   // are of a known payload type. SplitAudio() method is protected against
575   // sync-packets.
576   ret = payload_splitter_->SplitAudio(&packet_list, *decoder_database_);
577   if (ret != PayloadSplitter::kOK) {
578     LOG_FERR1(LS_WARNING, SplitAudio, packet_list.size());
579     PacketBuffer::DeleteAllPackets(&packet_list);
580     switch (ret) {
581       case PayloadSplitter::kUnknownPayloadType:
582         return kUnknownRtpPayloadType;
583       case PayloadSplitter::kFrameSplitError:
584         return kFrameSplitError;
585       default:
586         return kOtherError;
587     }
588   }
589 
590   // Update bandwidth estimate, if the packet is not sync-packet.
591   if (!packet_list.empty() && !packet_list.front()->sync_packet) {
592     // The list can be empty here if we got nothing but DTMF payloads.
593     AudioDecoder* decoder =
594         decoder_database_->GetDecoder(main_header.payloadType);
595     assert(decoder);  // Should always get a valid object, since we have
596                       // already checked that the payload types are known.
597     decoder->IncomingPacket(packet_list.front()->payload,
598                             packet_list.front()->payload_length,
599                             packet_list.front()->header.sequenceNumber,
600                             packet_list.front()->header.timestamp,
601                             receive_timestamp);
602   }
603 
604   // Insert packets in buffer.
605   int temp_bufsize = packet_buffer_->NumPacketsInBuffer();
606   ret = packet_buffer_->InsertPacketList(
607       &packet_list,
608       *decoder_database_,
609       &current_rtp_payload_type_,
610       &current_cng_rtp_payload_type_);
611   if (ret == PacketBuffer::kFlushed) {
612     // Reset DSP timestamp etc. if packet buffer flushed.
613     new_codec_ = true;
614     update_sample_rate_and_channels = true;
615     LOG_F(LS_WARNING) << "Packet buffer flushed";
616   } else if (ret != PacketBuffer::kOK) {
617     LOG_FERR1(LS_WARNING, InsertPacketList, packet_list.size());
618     PacketBuffer::DeleteAllPackets(&packet_list);
619     return kOtherError;
620   }
621   if (current_rtp_payload_type_ != 0xFF) {
622     const DecoderDatabase::DecoderInfo* dec_info =
623         decoder_database_->GetDecoderInfo(current_rtp_payload_type_);
624     if (!dec_info) {
625       assert(false);  // Already checked that the payload type is known.
626     }
627   }
628 
629   if (update_sample_rate_and_channels && !packet_buffer_->Empty()) {
630     // We do not use |current_rtp_payload_type_| to |set payload_type|, but
631     // get the next RTP header from |packet_buffer_| to obtain the payload type.
632     // The reason for it is the following corner case. If NetEq receives a
633     // CNG packet with a sample rate different than the current CNG then it
634     // flushes its buffer, assuming send codec must have been changed. However,
635     // payload type of the hypothetically new send codec is not known.
636     const RTPHeader* rtp_header = packet_buffer_->NextRtpHeader();
637     assert(rtp_header);
638     int payload_type = rtp_header->payloadType;
639     AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type);
640     assert(decoder);  // Payloads are already checked to be valid.
641     const DecoderDatabase::DecoderInfo* decoder_info =
642         decoder_database_->GetDecoderInfo(payload_type);
643     assert(decoder_info);
644     if (decoder_info->fs_hz != fs_hz_ ||
645         decoder->channels() != algorithm_buffer_->Channels())
646       SetSampleRateAndChannels(decoder_info->fs_hz, decoder->channels());
647   }
648 
649   // TODO(hlundin): Move this code to DelayManager class.
650   const DecoderDatabase::DecoderInfo* dec_info =
651           decoder_database_->GetDecoderInfo(main_header.payloadType);
652   assert(dec_info);  // Already checked that the payload type is known.
653   delay_manager_->LastDecoderType(dec_info->codec_type);
654   if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
655     // Calculate the total speech length carried in each packet.
656     temp_bufsize = packet_buffer_->NumPacketsInBuffer() - temp_bufsize;
657     temp_bufsize *= decoder_frame_length_;
658 
659     if ((temp_bufsize > 0) &&
660         (temp_bufsize != decision_logic_->packet_length_samples())) {
661       decision_logic_->set_packet_length_samples(temp_bufsize);
662       delay_manager_->SetPacketAudioLength((1000 * temp_bufsize) / fs_hz_);
663     }
664 
665     // Update statistics.
666     if ((int32_t) (main_header.timestamp - timestamp_) >= 0 &&
667         !new_codec_) {
668       // Only update statistics if incoming packet is not older than last played
669       // out packet, and if new codec flag is not set.
670       delay_manager_->Update(main_header.sequenceNumber, main_header.timestamp,
671                              fs_hz_);
672     }
673   } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
674     // This is first "normal" packet after CNG or DTMF.
675     // Reset packet time counter and measure time until next packet,
676     // but don't update statistics.
677     delay_manager_->set_last_pack_cng_or_dtmf(0);
678     delay_manager_->ResetPacketIatCount();
679   }
680   return 0;
681 }
682 
GetAudioInternal(size_t max_length,int16_t * output,int * samples_per_channel,int * num_channels)683 int NetEqImpl::GetAudioInternal(size_t max_length, int16_t* output,
684                                 int* samples_per_channel, int* num_channels) {
685   PacketList packet_list;
686   DtmfEvent dtmf_event;
687   Operations operation;
688   bool play_dtmf;
689   int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
690                                  &play_dtmf);
691   if (return_value != 0) {
692     LOG_FERR1(LS_WARNING, GetDecision, return_value);
693     assert(false);
694     last_mode_ = kModeError;
695     return return_value;
696   }
697   LOG(LS_VERBOSE) << "GetDecision returned operation=" << operation <<
698       " and " << packet_list.size() << " packet(s)";
699 
700   AudioDecoder::SpeechType speech_type;
701   int length = 0;
702   int decode_return_value = Decode(&packet_list, &operation,
703                                    &length, &speech_type);
704 
705   assert(vad_.get());
706   bool sid_frame_available =
707       (operation == kRfc3389Cng && !packet_list.empty());
708   vad_->Update(decoded_buffer_.get(), length, speech_type,
709                sid_frame_available, fs_hz_);
710 
711   algorithm_buffer_->Clear();
712   switch (operation) {
713     case kNormal: {
714       DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
715       break;
716     }
717     case kMerge: {
718       DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
719       break;
720     }
721     case kExpand: {
722       return_value = DoExpand(play_dtmf);
723       break;
724     }
725     case kAccelerate: {
726       return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type,
727                                   play_dtmf);
728       break;
729     }
730     case kPreemptiveExpand: {
731       return_value = DoPreemptiveExpand(decoded_buffer_.get(), length,
732                                         speech_type, play_dtmf);
733       break;
734     }
735     case kRfc3389Cng:
736     case kRfc3389CngNoPacket: {
737       return_value = DoRfc3389Cng(&packet_list, play_dtmf);
738       break;
739     }
740     case kCodecInternalCng: {
741       // This handles the case when there is no transmission and the decoder
742       // should produce internal comfort noise.
743       // TODO(hlundin): Write test for codec-internal CNG.
744       DoCodecInternalCng();
745       break;
746     }
747     case kDtmf: {
748       // TODO(hlundin): Write test for this.
749       return_value = DoDtmf(dtmf_event, &play_dtmf);
750       break;
751     }
752     case kAlternativePlc: {
753       // TODO(hlundin): Write test for this.
754       DoAlternativePlc(false);
755       break;
756     }
757     case kAlternativePlcIncreaseTimestamp: {
758       // TODO(hlundin): Write test for this.
759       DoAlternativePlc(true);
760       break;
761     }
762     case kAudioRepetitionIncreaseTimestamp: {
763       // TODO(hlundin): Write test for this.
764       sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
765       // Skipping break on purpose. Execution should move on into the
766       // next case.
767     }
768     case kAudioRepetition: {
769       // TODO(hlundin): Write test for this.
770       // Copy last |output_size_samples_| from |sync_buffer_| to
771       // |algorithm_buffer|.
772       algorithm_buffer_->PushBackFromIndex(
773           *sync_buffer_, sync_buffer_->Size() - output_size_samples_);
774       expand_->Reset();
775       break;
776     }
777     case kUndefined: {
778       LOG_F(LS_ERROR) << "Invalid operation kUndefined.";
779       assert(false);  // This should not happen.
780       last_mode_ = kModeError;
781       return kInvalidOperation;
782     }
783   }  // End of switch.
784   if (return_value < 0) {
785     return return_value;
786   }
787 
788   if (last_mode_ != kModeRfc3389Cng) {
789     comfort_noise_->Reset();
790   }
791 
792   // Copy from |algorithm_buffer| to |sync_buffer_|.
793   sync_buffer_->PushBack(*algorithm_buffer_);
794 
795   // Extract data from |sync_buffer_| to |output|.
796   size_t num_output_samples_per_channel = output_size_samples_;
797   size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels();
798   if (num_output_samples > max_length) {
799     LOG(LS_WARNING) << "Output array is too short. " << max_length << " < " <<
800         output_size_samples_ << " * " << sync_buffer_->Channels();
801     num_output_samples = max_length;
802     num_output_samples_per_channel = static_cast<int>(
803         max_length / sync_buffer_->Channels());
804   }
805   int samples_from_sync = static_cast<int>(
806       sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
807                                             output));
808   *num_channels = static_cast<int>(sync_buffer_->Channels());
809   LOG(LS_VERBOSE) << "Sync buffer (" << *num_channels << " channel(s)):" <<
810       " insert " << algorithm_buffer_->Size() << " samples, extract " <<
811       samples_from_sync << " samples";
812   if (samples_from_sync != output_size_samples_) {
813     LOG_F(LS_ERROR) << "samples_from_sync != output_size_samples_";
814     // TODO(minyue): treatment of under-run, filling zeros
815     memset(output, 0, num_output_samples * sizeof(int16_t));
816     *samples_per_channel = output_size_samples_;
817     return kSampleUnderrun;
818   }
819   *samples_per_channel = output_size_samples_;
820 
821   // Should always have overlap samples left in the |sync_buffer_|.
822   assert(sync_buffer_->FutureLength() >= expand_->overlap_length());
823 
824   if (play_dtmf) {
825     return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(), output);
826   }
827 
828   // Update the background noise parameters if last operation wrote data
829   // straight from the decoder to the |sync_buffer_|. That is, none of the
830   // operations that modify the signal can be followed by a parameter update.
831   if ((last_mode_ == kModeNormal) ||
832       (last_mode_ == kModeAccelerateFail) ||
833       (last_mode_ == kModePreemptiveExpandFail) ||
834       (last_mode_ == kModeRfc3389Cng) ||
835       (last_mode_ == kModeCodecInternalCng)) {
836     background_noise_->Update(*sync_buffer_, *vad_.get());
837   }
838 
839   if (operation == kDtmf) {
840     // DTMF data was written the end of |sync_buffer_|.
841     // Update index to end of DTMF data in |sync_buffer_|.
842     sync_buffer_->set_dtmf_index(sync_buffer_->Size());
843   }
844 
845   if (last_mode_ != kModeExpand) {
846     // If last operation was not expand, calculate the |playout_timestamp_| from
847     // the |sync_buffer_|. However, do not update the |playout_timestamp_| if it
848     // would be moved "backwards".
849     uint32_t temp_timestamp = sync_buffer_->end_timestamp() -
850         static_cast<uint32_t>(sync_buffer_->FutureLength());
851     if (static_cast<int32_t>(temp_timestamp - playout_timestamp_) > 0) {
852       playout_timestamp_ = temp_timestamp;
853     }
854   } else {
855     // Use dead reckoning to estimate the |playout_timestamp_|.
856     playout_timestamp_ += output_size_samples_;
857   }
858 
859   if (decode_return_value) return decode_return_value;
860   return return_value;
861 }
862 
GetDecision(Operations * operation,PacketList * packet_list,DtmfEvent * dtmf_event,bool * play_dtmf)863 int NetEqImpl::GetDecision(Operations* operation,
864                            PacketList* packet_list,
865                            DtmfEvent* dtmf_event,
866                            bool* play_dtmf) {
867   // Initialize output variables.
868   *play_dtmf = false;
869   *operation = kUndefined;
870 
871   // Increment time counters.
872   packet_buffer_->IncrementWaitingTimes();
873   stats_.IncreaseCounter(output_size_samples_, fs_hz_);
874 
875   assert(sync_buffer_.get());
876   uint32_t end_timestamp = sync_buffer_->end_timestamp();
877   if (!new_codec_) {
878     packet_buffer_->DiscardOldPackets(end_timestamp);
879   }
880   const RTPHeader* header = packet_buffer_->NextRtpHeader();
881 
882   if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
883     // Because of timestamp peculiarities, we have to "manually" disallow using
884     // a CNG packet with the same timestamp as the one that was last played.
885     // This can happen when using redundancy and will cause the timing to shift.
886     while (header && decoder_database_->IsComfortNoise(header->payloadType) &&
887            (end_timestamp >= header->timestamp ||
888             end_timestamp + decision_logic_->generated_noise_samples() >
889                 header->timestamp)) {
890       // Don't use this packet, discard it.
891       if (packet_buffer_->DiscardNextPacket() != PacketBuffer::kOK) {
892         assert(false);  // Must be ok by design.
893       }
894       // Check buffer again.
895       if (!new_codec_) {
896         packet_buffer_->DiscardOldPackets(end_timestamp);
897       }
898       header = packet_buffer_->NextRtpHeader();
899     }
900   }
901 
902   assert(expand_.get());
903   const int samples_left = static_cast<int>(sync_buffer_->FutureLength() -
904       expand_->overlap_length());
905   if (last_mode_ == kModeAccelerateSuccess ||
906       last_mode_ == kModeAccelerateLowEnergy ||
907       last_mode_ == kModePreemptiveExpandSuccess ||
908       last_mode_ == kModePreemptiveExpandLowEnergy) {
909     // Subtract (samples_left + output_size_samples_) from sampleMemory.
910     decision_logic_->AddSampleMemory(-(samples_left + output_size_samples_));
911   }
912 
913   // Check if it is time to play a DTMF event.
914   if (dtmf_buffer_->GetEvent(end_timestamp +
915                              decision_logic_->generated_noise_samples(),
916                              dtmf_event)) {
917     *play_dtmf = true;
918   }
919 
920   // Get instruction.
921   assert(sync_buffer_.get());
922   assert(expand_.get());
923   *operation = decision_logic_->GetDecision(*sync_buffer_,
924                                             *expand_,
925                                             decoder_frame_length_,
926                                             header,
927                                             last_mode_,
928                                             *play_dtmf,
929                                             &reset_decoder_);
930 
931   // Check if we already have enough samples in the |sync_buffer_|. If so,
932   // change decision to normal, unless the decision was merge, accelerate, or
933   // preemptive expand.
934   if (samples_left >= output_size_samples_ &&
935       *operation != kMerge &&
936       *operation != kAccelerate &&
937       *operation != kPreemptiveExpand) {
938     *operation = kNormal;
939     return 0;
940   }
941 
942   decision_logic_->ExpandDecision(*operation);
943 
944   // Check conditions for reset.
945   if (new_codec_ || *operation == kUndefined) {
946     // The only valid reason to get kUndefined is that new_codec_ is set.
947     assert(new_codec_);
948     if (*play_dtmf && !header) {
949       timestamp_ = dtmf_event->timestamp;
950     } else {
951       assert(header);
952       if (!header) {
953         LOG_F(LS_ERROR) << "Packet missing where it shouldn't.";
954         return -1;
955       }
956       timestamp_ = header->timestamp;
957       if (*operation == kRfc3389CngNoPacket
958 #ifndef LEGACY_BITEXACT
959           // Without this check, it can happen that a non-CNG packet is sent to
960           // the CNG decoder as if it was a SID frame. This is clearly a bug,
961           // but is kept for now to maintain bit-exactness with the test
962           // vectors.
963           && decoder_database_->IsComfortNoise(header->payloadType)
964 #endif
965       ) {
966         // Change decision to CNG packet, since we do have a CNG packet, but it
967         // was considered too early to use. Now, use it anyway.
968         *operation = kRfc3389Cng;
969       } else if (*operation != kRfc3389Cng) {
970         *operation = kNormal;
971       }
972     }
973     // Adjust |sync_buffer_| timestamp before setting |end_timestamp| to the
974     // new value.
975     sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp);
976     end_timestamp = timestamp_;
977     new_codec_ = false;
978     decision_logic_->SoftReset();
979     buffer_level_filter_->Reset();
980     delay_manager_->Reset();
981     stats_.ResetMcu();
982   }
983 
984   int required_samples = output_size_samples_;
985   const int samples_10_ms = 80 * fs_mult_;
986   const int samples_20_ms = 2 * samples_10_ms;
987   const int samples_30_ms = 3 * samples_10_ms;
988 
989   switch (*operation) {
990     case kExpand: {
991       timestamp_ = end_timestamp;
992       return 0;
993     }
994     case kRfc3389CngNoPacket:
995     case kCodecInternalCng: {
996       return 0;
997     }
998     case kDtmf: {
999       // TODO(hlundin): Write test for this.
1000       // Update timestamp.
1001       timestamp_ = end_timestamp;
1002       if (decision_logic_->generated_noise_samples() > 0 &&
1003           last_mode_ != kModeDtmf) {
1004         // Make a jump in timestamp due to the recently played comfort noise.
1005         uint32_t timestamp_jump = decision_logic_->generated_noise_samples();
1006         sync_buffer_->IncreaseEndTimestamp(timestamp_jump);
1007         timestamp_ += timestamp_jump;
1008       }
1009       decision_logic_->set_generated_noise_samples(0);
1010       return 0;
1011     }
1012     case kAccelerate: {
1013       // In order to do a accelerate we need at least 30 ms of audio data.
1014       if (samples_left >= samples_30_ms) {
1015         // Already have enough data, so we do not need to extract any more.
1016         decision_logic_->set_sample_memory(samples_left);
1017         decision_logic_->set_prev_time_scale(true);
1018         return 0;
1019       } else if (samples_left >= samples_10_ms &&
1020           decoder_frame_length_ >= samples_30_ms) {
1021         // Avoid decoding more data as it might overflow the playout buffer.
1022         *operation = kNormal;
1023         return 0;
1024       } else if (samples_left < samples_20_ms &&
1025           decoder_frame_length_ < samples_30_ms) {
1026         // Build up decoded data by decoding at least 20 ms of audio data. Do
1027         // not perform accelerate yet, but wait until we only need to do one
1028         // decoding.
1029         required_samples = 2 * output_size_samples_;
1030         *operation = kNormal;
1031       }
1032       // If none of the above is true, we have one of two possible situations:
1033       // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or
1034       // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms.
1035       // In either case, we move on with the accelerate decision, and decode one
1036       // frame now.
1037       break;
1038     }
1039     case kPreemptiveExpand: {
1040       // In order to do a preemptive expand we need at least 30 ms of decoded
1041       // audio data.
1042       if ((samples_left >= samples_30_ms) ||
1043           (samples_left >= samples_10_ms &&
1044               decoder_frame_length_ >= samples_30_ms)) {
1045         // Already have enough data, so we do not need to extract any more.
1046         // Or, avoid decoding more data as it might overflow the playout buffer.
1047         // Still try preemptive expand, though.
1048         decision_logic_->set_sample_memory(samples_left);
1049         decision_logic_->set_prev_time_scale(true);
1050         return 0;
1051       }
1052       if (samples_left < samples_20_ms &&
1053           decoder_frame_length_ < samples_30_ms) {
1054         // Build up decoded data by decoding at least 20 ms of audio data.
1055         // Still try to perform preemptive expand.
1056         required_samples = 2 * output_size_samples_;
1057       }
1058       // Move on with the preemptive expand decision.
1059       break;
1060     }
1061     case kMerge: {
1062       required_samples =
1063           std::max(merge_->RequiredFutureSamples(), required_samples);
1064       break;
1065     }
1066     default: {
1067       // Do nothing.
1068     }
1069   }
1070 
1071   // Get packets from buffer.
1072   int extracted_samples = 0;
1073   if (header &&
1074       *operation != kAlternativePlc &&
1075       *operation != kAlternativePlcIncreaseTimestamp &&
1076       *operation != kAudioRepetition &&
1077       *operation != kAudioRepetitionIncreaseTimestamp) {
1078     sync_buffer_->IncreaseEndTimestamp(header->timestamp - end_timestamp);
1079     if (decision_logic_->CngOff()) {
1080       // Adjustment of timestamp only corresponds to an actual packet loss
1081       // if comfort noise is not played. If comfort noise was just played,
1082       // this adjustment of timestamp is only done to get back in sync with the
1083       // stream timestamp; no loss to report.
1084       stats_.LostSamples(header->timestamp - end_timestamp);
1085     }
1086 
1087     if (*operation != kRfc3389Cng) {
1088       // We are about to decode and use a non-CNG packet.
1089       decision_logic_->SetCngOff();
1090     }
1091     // Reset CNG timestamp as a new packet will be delivered.
1092     // (Also if this is a CNG packet, since playedOutTS is updated.)
1093     decision_logic_->set_generated_noise_samples(0);
1094 
1095     extracted_samples = ExtractPackets(required_samples, packet_list);
1096     if (extracted_samples < 0) {
1097       LOG_F(LS_WARNING) << "Failed to extract packets from buffer.";
1098       return kPacketBufferCorruption;
1099     }
1100   }
1101 
1102   if (*operation == kAccelerate ||
1103       *operation == kPreemptiveExpand) {
1104     decision_logic_->set_sample_memory(samples_left + extracted_samples);
1105     decision_logic_->set_prev_time_scale(true);
1106   }
1107 
1108   if (*operation == kAccelerate) {
1109     // Check that we have enough data (30ms) to do accelerate.
1110     if (extracted_samples + samples_left < samples_30_ms) {
1111       // TODO(hlundin): Write test for this.
1112       // Not enough, do normal operation instead.
1113       *operation = kNormal;
1114     }
1115   }
1116 
1117   timestamp_ = end_timestamp;
1118   return 0;
1119 }
1120 
Decode(PacketList * packet_list,Operations * operation,int * decoded_length,AudioDecoder::SpeechType * speech_type)1121 int NetEqImpl::Decode(PacketList* packet_list, Operations* operation,
1122                       int* decoded_length,
1123                       AudioDecoder::SpeechType* speech_type) {
1124   *speech_type = AudioDecoder::kSpeech;
1125   AudioDecoder* decoder = NULL;
1126   if (!packet_list->empty()) {
1127     const Packet* packet = packet_list->front();
1128     int payload_type = packet->header.payloadType;
1129     if (!decoder_database_->IsComfortNoise(payload_type)) {
1130       decoder = decoder_database_->GetDecoder(payload_type);
1131       assert(decoder);
1132       if (!decoder) {
1133         LOG_FERR1(LS_WARNING, GetDecoder, payload_type);
1134         PacketBuffer::DeleteAllPackets(packet_list);
1135         return kDecoderNotFound;
1136       }
1137       bool decoder_changed;
1138       decoder_database_->SetActiveDecoder(payload_type, &decoder_changed);
1139       if (decoder_changed) {
1140         // We have a new decoder. Re-init some values.
1141         const DecoderDatabase::DecoderInfo* decoder_info = decoder_database_
1142             ->GetDecoderInfo(payload_type);
1143         assert(decoder_info);
1144         if (!decoder_info) {
1145           LOG_FERR1(LS_WARNING, GetDecoderInfo, payload_type);
1146           PacketBuffer::DeleteAllPackets(packet_list);
1147           return kDecoderNotFound;
1148         }
1149         // If sampling rate or number of channels has changed, we need to make
1150         // a reset.
1151         if (decoder_info->fs_hz != fs_hz_ ||
1152             decoder->channels() != algorithm_buffer_->Channels()) {
1153           // TODO(tlegrand): Add unittest to cover this event.
1154           SetSampleRateAndChannels(decoder_info->fs_hz, decoder->channels());
1155         }
1156         sync_buffer_->set_end_timestamp(timestamp_);
1157         playout_timestamp_ = timestamp_;
1158       }
1159     }
1160   }
1161 
1162   if (reset_decoder_) {
1163     // TODO(hlundin): Write test for this.
1164     // Reset decoder.
1165     if (decoder) {
1166       decoder->Init();
1167     }
1168     // Reset comfort noise decoder.
1169     AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1170     if (cng_decoder) {
1171       cng_decoder->Init();
1172     }
1173     reset_decoder_ = false;
1174   }
1175 
1176 #ifdef LEGACY_BITEXACT
1177   // Due to a bug in old SignalMCU, it could happen that CNG operation was
1178   // decided, but a speech packet was provided. The speech packet will be used
1179   // to update the comfort noise decoder, as if it was a SID frame, which is
1180   // clearly wrong.
1181   if (*operation == kRfc3389Cng) {
1182     return 0;
1183   }
1184 #endif
1185 
1186   *decoded_length = 0;
1187   // Update codec-internal PLC state.
1188   if ((*operation == kMerge) && decoder && decoder->HasDecodePlc()) {
1189     decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]);
1190   }
1191 
1192   int return_value = DecodeLoop(packet_list, operation, decoder,
1193                                 decoded_length, speech_type);
1194 
1195   if (*decoded_length < 0) {
1196     // Error returned from the decoder.
1197     *decoded_length = 0;
1198     sync_buffer_->IncreaseEndTimestamp(decoder_frame_length_);
1199     int error_code = 0;
1200     if (decoder)
1201       error_code = decoder->ErrorCode();
1202     if (error_code != 0) {
1203       // Got some error code from the decoder.
1204       decoder_error_code_ = error_code;
1205       return_value = kDecoderErrorCode;
1206     } else {
1207       // Decoder does not implement error codes. Return generic error.
1208       return_value = kOtherDecoderError;
1209     }
1210     LOG_FERR2(LS_WARNING, DecodeLoop, error_code, packet_list->size());
1211     *operation = kExpand;  // Do expansion to get data instead.
1212   }
1213   if (*speech_type != AudioDecoder::kComfortNoise) {
1214     // Don't increment timestamp if codec returned CNG speech type
1215     // since in this case, the we will increment the CNGplayedTS counter.
1216     // Increase with number of samples per channel.
1217     assert(*decoded_length == 0 ||
1218            (decoder && decoder->channels() == sync_buffer_->Channels()));
1219     sync_buffer_->IncreaseEndTimestamp(
1220         *decoded_length / static_cast<int>(sync_buffer_->Channels()));
1221   }
1222   return return_value;
1223 }
1224 
DecodeLoop(PacketList * packet_list,Operations * operation,AudioDecoder * decoder,int * decoded_length,AudioDecoder::SpeechType * speech_type)1225 int NetEqImpl::DecodeLoop(PacketList* packet_list, Operations* operation,
1226                           AudioDecoder* decoder, int* decoded_length,
1227                           AudioDecoder::SpeechType* speech_type) {
1228   Packet* packet = NULL;
1229   if (!packet_list->empty()) {
1230     packet = packet_list->front();
1231   }
1232   // Do decoding.
1233   while (packet &&
1234       !decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1235     assert(decoder);  // At this point, we must have a decoder object.
1236     // The number of channels in the |sync_buffer_| should be the same as the
1237     // number decoder channels.
1238     assert(sync_buffer_->Channels() == decoder->channels());
1239     assert(decoded_buffer_length_ >= kMaxFrameSize * decoder->channels());
1240     assert(*operation == kNormal || *operation == kAccelerate ||
1241            *operation == kMerge || *operation == kPreemptiveExpand);
1242     packet_list->pop_front();
1243     int payload_length = packet->payload_length;
1244     int16_t decode_length;
1245     if (packet->sync_packet) {
1246       // Decode to silence with the same frame size as the last decode.
1247       LOG(LS_VERBOSE) << "Decoding sync-packet: " <<
1248           " ts=" << packet->header.timestamp <<
1249           ", sn=" << packet->header.sequenceNumber <<
1250           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1251           ", ssrc=" << packet->header.ssrc <<
1252           ", len=" << packet->payload_length;
1253       memset(&decoded_buffer_[*decoded_length], 0, decoder_frame_length_ *
1254              decoder->channels() * sizeof(decoded_buffer_[0]));
1255       decode_length = decoder_frame_length_;
1256     } else if (!packet->primary) {
1257       // This is a redundant payload; call the special decoder method.
1258       LOG(LS_VERBOSE) << "Decoding packet (redundant):" <<
1259           " ts=" << packet->header.timestamp <<
1260           ", sn=" << packet->header.sequenceNumber <<
1261           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1262           ", ssrc=" << packet->header.ssrc <<
1263           ", len=" << packet->payload_length;
1264       decode_length = decoder->DecodeRedundant(
1265           packet->payload, packet->payload_length,
1266           &decoded_buffer_[*decoded_length], speech_type);
1267     } else {
1268       LOG(LS_VERBOSE) << "Decoding packet: ts=" << packet->header.timestamp <<
1269           ", sn=" << packet->header.sequenceNumber <<
1270           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1271           ", ssrc=" << packet->header.ssrc <<
1272           ", len=" << packet->payload_length;
1273       decode_length = decoder->Decode(packet->payload,
1274                                       packet->payload_length,
1275                                       &decoded_buffer_[*decoded_length],
1276                                       speech_type);
1277     }
1278 
1279     delete[] packet->payload;
1280     delete packet;
1281     packet = NULL;
1282     if (decode_length > 0) {
1283       *decoded_length += decode_length;
1284       // Update |decoder_frame_length_| with number of samples per channel.
1285       decoder_frame_length_ = decode_length /
1286           static_cast<int>(decoder->channels());
1287       LOG(LS_VERBOSE) << "Decoded " << decode_length << " samples (" <<
1288           decoder->channels() << " channel(s) -> " << decoder_frame_length_ <<
1289           " samples per channel)";
1290     } else if (decode_length < 0) {
1291       // Error.
1292       LOG_FERR2(LS_WARNING, Decode, decode_length, payload_length);
1293       *decoded_length = -1;
1294       PacketBuffer::DeleteAllPackets(packet_list);
1295       break;
1296     }
1297     if (*decoded_length > static_cast<int>(decoded_buffer_length_)) {
1298       // Guard against overflow.
1299       LOG_F(LS_WARNING) << "Decoded too much.";
1300       PacketBuffer::DeleteAllPackets(packet_list);
1301       return kDecodedTooMuch;
1302     }
1303     if (!packet_list->empty()) {
1304       packet = packet_list->front();
1305     } else {
1306       packet = NULL;
1307     }
1308   }  // End of decode loop.
1309 
1310   // If the list is not empty at this point, either a decoding error terminated
1311   // the while-loop, or list must hold exactly one CNG packet.
1312   assert(packet_list->empty() || *decoded_length < 0 ||
1313          (packet_list->size() == 1 && packet &&
1314              decoder_database_->IsComfortNoise(packet->header.payloadType)));
1315   return 0;
1316 }
1317 
DoNormal(const int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1318 void NetEqImpl::DoNormal(const int16_t* decoded_buffer, size_t decoded_length,
1319                          AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1320   assert(normal_.get());
1321   assert(mute_factor_array_.get());
1322   normal_->Process(decoded_buffer, decoded_length, last_mode_,
1323                    mute_factor_array_.get(), algorithm_buffer_.get());
1324   if (decoded_length != 0) {
1325     last_mode_ = kModeNormal;
1326   }
1327 
1328   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1329   if ((speech_type == AudioDecoder::kComfortNoise)
1330       || ((last_mode_ == kModeCodecInternalCng)
1331           && (decoded_length == 0))) {
1332     // TODO(hlundin): Remove second part of || statement above.
1333     last_mode_ = kModeCodecInternalCng;
1334   }
1335 
1336   if (!play_dtmf) {
1337     dtmf_tone_generator_->Reset();
1338   }
1339 }
1340 
DoMerge(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1341 void NetEqImpl::DoMerge(int16_t* decoded_buffer, size_t decoded_length,
1342                         AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1343   assert(mute_factor_array_.get());
1344   assert(merge_.get());
1345   int new_length = merge_->Process(decoded_buffer, decoded_length,
1346                                    mute_factor_array_.get(),
1347                                    algorithm_buffer_.get());
1348 
1349   // Update in-call and post-call statistics.
1350   if (expand_->MuteFactor(0) == 0) {
1351     // Expand generates only noise.
1352     stats_.ExpandedNoiseSamples(new_length - static_cast<int>(decoded_length));
1353   } else {
1354     // Expansion generates more than only noise.
1355     stats_.ExpandedVoiceSamples(new_length - static_cast<int>(decoded_length));
1356   }
1357 
1358   last_mode_ = kModeMerge;
1359   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1360   if (speech_type == AudioDecoder::kComfortNoise) {
1361     last_mode_ = kModeCodecInternalCng;
1362   }
1363   expand_->Reset();
1364   if (!play_dtmf) {
1365     dtmf_tone_generator_->Reset();
1366   }
1367 }
1368 
DoExpand(bool play_dtmf)1369 int NetEqImpl::DoExpand(bool play_dtmf) {
1370   while ((sync_buffer_->FutureLength() - expand_->overlap_length()) <
1371       static_cast<size_t>(output_size_samples_)) {
1372     algorithm_buffer_->Clear();
1373     int return_value = expand_->Process(algorithm_buffer_.get());
1374     int length = static_cast<int>(algorithm_buffer_->Size());
1375 
1376     // Update in-call and post-call statistics.
1377     if (expand_->MuteFactor(0) == 0) {
1378       // Expand operation generates only noise.
1379       stats_.ExpandedNoiseSamples(length);
1380     } else {
1381       // Expand operation generates more than only noise.
1382       stats_.ExpandedVoiceSamples(length);
1383     }
1384 
1385     last_mode_ = kModeExpand;
1386 
1387     if (return_value < 0) {
1388       return return_value;
1389     }
1390 
1391     sync_buffer_->PushBack(*algorithm_buffer_);
1392     algorithm_buffer_->Clear();
1393   }
1394   if (!play_dtmf) {
1395     dtmf_tone_generator_->Reset();
1396   }
1397   return 0;
1398 }
1399 
DoAccelerate(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1400 int NetEqImpl::DoAccelerate(int16_t* decoded_buffer, size_t decoded_length,
1401                             AudioDecoder::SpeechType speech_type,
1402                             bool play_dtmf) {
1403   const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1404   size_t borrowed_samples_per_channel = 0;
1405   size_t num_channels = algorithm_buffer_->Channels();
1406   size_t decoded_length_per_channel = decoded_length / num_channels;
1407   if (decoded_length_per_channel < required_samples) {
1408     // Must move data from the |sync_buffer_| in order to get 30 ms.
1409     borrowed_samples_per_channel = static_cast<int>(required_samples -
1410         decoded_length_per_channel);
1411     memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1412             decoded_buffer,
1413             sizeof(int16_t) * decoded_length);
1414     sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1415                                          decoded_buffer);
1416     decoded_length = required_samples * num_channels;
1417   }
1418 
1419   int16_t samples_removed;
1420   Accelerate::ReturnCodes return_code = accelerate_->Process(
1421       decoded_buffer, decoded_length, algorithm_buffer_.get(),
1422       &samples_removed);
1423   stats_.AcceleratedSamples(samples_removed);
1424   switch (return_code) {
1425     case Accelerate::kSuccess:
1426       last_mode_ = kModeAccelerateSuccess;
1427       break;
1428     case Accelerate::kSuccessLowEnergy:
1429       last_mode_ = kModeAccelerateLowEnergy;
1430       break;
1431     case Accelerate::kNoStretch:
1432       last_mode_ = kModeAccelerateFail;
1433       break;
1434     case Accelerate::kError:
1435       // TODO(hlundin): Map to kModeError instead?
1436       last_mode_ = kModeAccelerateFail;
1437       return kAccelerateError;
1438   }
1439 
1440   if (borrowed_samples_per_channel > 0) {
1441     // Copy borrowed samples back to the |sync_buffer_|.
1442     size_t length = algorithm_buffer_->Size();
1443     if (length < borrowed_samples_per_channel) {
1444       // This destroys the beginning of the buffer, but will not cause any
1445       // problems.
1446       sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1447                                    sync_buffer_->Size() -
1448                                    borrowed_samples_per_channel);
1449       sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length);
1450       algorithm_buffer_->PopFront(length);
1451       assert(algorithm_buffer_->Empty());
1452     } else {
1453       sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1454                                    borrowed_samples_per_channel,
1455                                    sync_buffer_->Size() -
1456                                    borrowed_samples_per_channel);
1457       algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1458     }
1459   }
1460 
1461   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1462   if (speech_type == AudioDecoder::kComfortNoise) {
1463     last_mode_ = kModeCodecInternalCng;
1464   }
1465   if (!play_dtmf) {
1466     dtmf_tone_generator_->Reset();
1467   }
1468   expand_->Reset();
1469   return 0;
1470 }
1471 
DoPreemptiveExpand(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1472 int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer,
1473                                   size_t decoded_length,
1474                                   AudioDecoder::SpeechType speech_type,
1475                                   bool play_dtmf) {
1476   const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1477   size_t num_channels = algorithm_buffer_->Channels();
1478   int borrowed_samples_per_channel = 0;
1479   int old_borrowed_samples_per_channel = 0;
1480   size_t decoded_length_per_channel = decoded_length / num_channels;
1481   if (decoded_length_per_channel < required_samples) {
1482     // Must move data from the |sync_buffer_| in order to get 30 ms.
1483     borrowed_samples_per_channel = static_cast<int>(required_samples -
1484         decoded_length_per_channel);
1485     // Calculate how many of these were already played out.
1486     old_borrowed_samples_per_channel = static_cast<int>(
1487         borrowed_samples_per_channel - sync_buffer_->FutureLength());
1488     old_borrowed_samples_per_channel = std::max(
1489         0, old_borrowed_samples_per_channel);
1490     memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1491             decoded_buffer,
1492             sizeof(int16_t) * decoded_length);
1493     sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1494                                          decoded_buffer);
1495     decoded_length = required_samples * num_channels;
1496   }
1497 
1498   int16_t samples_added;
1499   PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process(
1500       decoded_buffer, static_cast<int>(decoded_length),
1501       old_borrowed_samples_per_channel,
1502       algorithm_buffer_.get(), &samples_added);
1503   stats_.PreemptiveExpandedSamples(samples_added);
1504   switch (return_code) {
1505     case PreemptiveExpand::kSuccess:
1506       last_mode_ = kModePreemptiveExpandSuccess;
1507       break;
1508     case PreemptiveExpand::kSuccessLowEnergy:
1509       last_mode_ = kModePreemptiveExpandLowEnergy;
1510       break;
1511     case PreemptiveExpand::kNoStretch:
1512       last_mode_ = kModePreemptiveExpandFail;
1513       break;
1514     case PreemptiveExpand::kError:
1515       // TODO(hlundin): Map to kModeError instead?
1516       last_mode_ = kModePreemptiveExpandFail;
1517       return kPreemptiveExpandError;
1518   }
1519 
1520   if (borrowed_samples_per_channel > 0) {
1521     // Copy borrowed samples back to the |sync_buffer_|.
1522     sync_buffer_->ReplaceAtIndex(
1523         *algorithm_buffer_, borrowed_samples_per_channel,
1524         sync_buffer_->Size() - borrowed_samples_per_channel);
1525     algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1526   }
1527 
1528   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1529   if (speech_type == AudioDecoder::kComfortNoise) {
1530     last_mode_ = kModeCodecInternalCng;
1531   }
1532   if (!play_dtmf) {
1533     dtmf_tone_generator_->Reset();
1534   }
1535   expand_->Reset();
1536   return 0;
1537 }
1538 
DoRfc3389Cng(PacketList * packet_list,bool play_dtmf)1539 int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) {
1540   if (!packet_list->empty()) {
1541     // Must have exactly one SID frame at this point.
1542     assert(packet_list->size() == 1);
1543     Packet* packet = packet_list->front();
1544     packet_list->pop_front();
1545     if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1546 #ifdef LEGACY_BITEXACT
1547       // This can happen due to a bug in GetDecision. Change the payload type
1548       // to a CNG type, and move on. Note that this means that we are in fact
1549       // sending a non-CNG payload to the comfort noise decoder for decoding.
1550       // Clearly wrong, but will maintain bit-exactness with legacy.
1551       if (fs_hz_ == 8000) {
1552         packet->header.payloadType =
1553             decoder_database_->GetRtpPayloadType(kDecoderCNGnb);
1554       } else if (fs_hz_ == 16000) {
1555         packet->header.payloadType =
1556             decoder_database_->GetRtpPayloadType(kDecoderCNGwb);
1557       } else if (fs_hz_ == 32000) {
1558         packet->header.payloadType =
1559             decoder_database_->GetRtpPayloadType(kDecoderCNGswb32kHz);
1560       } else if (fs_hz_ == 48000) {
1561         packet->header.payloadType =
1562             decoder_database_->GetRtpPayloadType(kDecoderCNGswb48kHz);
1563       }
1564       assert(decoder_database_->IsComfortNoise(packet->header.payloadType));
1565 #else
1566       LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG.";
1567       return kOtherError;
1568 #endif
1569     }
1570     // UpdateParameters() deletes |packet|.
1571     if (comfort_noise_->UpdateParameters(packet) ==
1572         ComfortNoise::kInternalError) {
1573       LOG_FERR0(LS_WARNING, UpdateParameters);
1574       algorithm_buffer_->Zeros(output_size_samples_);
1575       return -comfort_noise_->internal_error_code();
1576     }
1577   }
1578   int cn_return = comfort_noise_->Generate(output_size_samples_,
1579                                            algorithm_buffer_.get());
1580   expand_->Reset();
1581   last_mode_ = kModeRfc3389Cng;
1582   if (!play_dtmf) {
1583     dtmf_tone_generator_->Reset();
1584   }
1585   if (cn_return == ComfortNoise::kInternalError) {
1586     LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1587     decoder_error_code_ = comfort_noise_->internal_error_code();
1588     return kComfortNoiseErrorCode;
1589   } else if (cn_return == ComfortNoise::kUnknownPayloadType) {
1590     LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1591     return kUnknownRtpPayloadType;
1592   }
1593   return 0;
1594 }
1595 
DoCodecInternalCng()1596 void NetEqImpl::DoCodecInternalCng() {
1597   int length = 0;
1598   // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1599   int16_t decoded_buffer[kMaxFrameSize];
1600   AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1601   if (decoder) {
1602     const uint8_t* dummy_payload = NULL;
1603     AudioDecoder::SpeechType speech_type;
1604     length = decoder->Decode(dummy_payload, 0, decoded_buffer, &speech_type);
1605   }
1606   assert(mute_factor_array_.get());
1607   normal_->Process(decoded_buffer, length, last_mode_, mute_factor_array_.get(),
1608                    algorithm_buffer_.get());
1609   last_mode_ = kModeCodecInternalCng;
1610   expand_->Reset();
1611 }
1612 
DoDtmf(const DtmfEvent & dtmf_event,bool * play_dtmf)1613 int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) {
1614   // This block of the code and the block further down, handling |dtmf_switch|
1615   // are commented out. Otherwise playing out-of-band DTMF would fail in VoE
1616   // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is
1617   // equivalent to |dtmf_switch| always be false.
1618   //
1619   // See http://webrtc-codereview.appspot.com/1195004/ for discussion
1620   // On this issue. This change might cause some glitches at the point of
1621   // switch from audio to DTMF. Issue 1545 is filed to track this.
1622   //
1623   //  bool dtmf_switch = false;
1624   //  if ((last_mode_ != kModeDtmf) && dtmf_tone_generator_->initialized()) {
1625   //    // Special case; see below.
1626   //    // We must catch this before calling Generate, since |initialized| is
1627   //    // modified in that call.
1628   //    dtmf_switch = true;
1629   //  }
1630 
1631   int dtmf_return_value = 0;
1632   if (!dtmf_tone_generator_->initialized()) {
1633     // Initialize if not already done.
1634     dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1635                                                    dtmf_event.volume);
1636   }
1637 
1638   if (dtmf_return_value == 0) {
1639     // Generate DTMF signal.
1640     dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_,
1641                                                        algorithm_buffer_.get());
1642   }
1643 
1644   if (dtmf_return_value < 0) {
1645     algorithm_buffer_->Zeros(output_size_samples_);
1646     return dtmf_return_value;
1647   }
1648 
1649   //  if (dtmf_switch) {
1650   //    // This is the special case where the previous operation was DTMF
1651   //    // overdub, but the current instruction is "regular" DTMF. We must make
1652   //    // sure that the DTMF does not have any discontinuities. The first DTMF
1653   //    // sample that we generate now must be played out immediately, therefore
1654   //    // it must be copied to the speech buffer.
1655   //    // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and
1656   //    // verify correct operation.
1657   //    assert(false);
1658   //    // Must generate enough data to replace all of the |sync_buffer_|
1659   //    // "future".
1660   //    int required_length = sync_buffer_->FutureLength();
1661   //    assert(dtmf_tone_generator_->initialized());
1662   //    dtmf_return_value = dtmf_tone_generator_->Generate(required_length,
1663   //                                                       algorithm_buffer_);
1664   //    assert((size_t) required_length == algorithm_buffer_->Size());
1665   //    if (dtmf_return_value < 0) {
1666   //      algorithm_buffer_->Zeros(output_size_samples_);
1667   //      return dtmf_return_value;
1668   //    }
1669   //
1670   //    // Overwrite the "future" part of the speech buffer with the new DTMF
1671   //    // data.
1672   //    // TODO(hlundin): It seems that this overwriting has gone lost.
1673   //    // Not adapted for multi-channel yet.
1674   //    assert(algorithm_buffer_->Channels() == 1);
1675   //    if (algorithm_buffer_->Channels() != 1) {
1676   //      LOG(LS_WARNING) << "DTMF not supported for more than one channel";
1677   //      return kStereoNotSupported;
1678   //    }
1679   //    // Shuffle the remaining data to the beginning of algorithm buffer.
1680   //    algorithm_buffer_->PopFront(sync_buffer_->FutureLength());
1681   //  }
1682 
1683   sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
1684   expand_->Reset();
1685   last_mode_ = kModeDtmf;
1686 
1687   // Set to false because the DTMF is already in the algorithm buffer.
1688   *play_dtmf = false;
1689   return 0;
1690 }
1691 
DoAlternativePlc(bool increase_timestamp)1692 void NetEqImpl::DoAlternativePlc(bool increase_timestamp) {
1693   AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1694   int length;
1695   if (decoder && decoder->HasDecodePlc()) {
1696     // Use the decoder's packet-loss concealment.
1697     // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1698     int16_t decoded_buffer[kMaxFrameSize];
1699     length = decoder->DecodePlc(1, decoded_buffer);
1700     if (length > 0) {
1701       algorithm_buffer_->PushBackInterleaved(decoded_buffer, length);
1702     } else {
1703       length = 0;
1704     }
1705   } else {
1706     // Do simple zero-stuffing.
1707     length = output_size_samples_;
1708     algorithm_buffer_->Zeros(length);
1709     // By not advancing the timestamp, NetEq inserts samples.
1710     stats_.AddZeros(length);
1711   }
1712   if (increase_timestamp) {
1713     sync_buffer_->IncreaseEndTimestamp(length);
1714   }
1715   expand_->Reset();
1716 }
1717 
DtmfOverdub(const DtmfEvent & dtmf_event,size_t num_channels,int16_t * output) const1718 int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels,
1719                            int16_t* output) const {
1720   size_t out_index = 0;
1721   int overdub_length = output_size_samples_;  // Default value.
1722 
1723   if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) {
1724     // Special operation for transition from "DTMF only" to "DTMF overdub".
1725     out_index = std::min(
1726         sync_buffer_->dtmf_index() - sync_buffer_->next_index(),
1727         static_cast<size_t>(output_size_samples_));
1728     overdub_length = output_size_samples_ - static_cast<int>(out_index);
1729   }
1730 
1731   AudioMultiVector dtmf_output(num_channels);
1732   int dtmf_return_value = 0;
1733   if (!dtmf_tone_generator_->initialized()) {
1734     dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1735                                                    dtmf_event.volume);
1736   }
1737   if (dtmf_return_value == 0) {
1738     dtmf_return_value = dtmf_tone_generator_->Generate(overdub_length,
1739                                                        &dtmf_output);
1740     assert((size_t) overdub_length == dtmf_output.Size());
1741   }
1742   dtmf_output.ReadInterleaved(overdub_length, &output[out_index]);
1743   return dtmf_return_value < 0 ? dtmf_return_value : 0;
1744 }
1745 
ExtractPackets(int required_samples,PacketList * packet_list)1746 int NetEqImpl::ExtractPackets(int required_samples, PacketList* packet_list) {
1747   bool first_packet = true;
1748   uint8_t prev_payload_type = 0;
1749   uint32_t prev_timestamp = 0;
1750   uint16_t prev_sequence_number = 0;
1751   bool next_packet_available = false;
1752 
1753   const RTPHeader* header = packet_buffer_->NextRtpHeader();
1754   assert(header);
1755   if (!header) {
1756     return -1;
1757   }
1758   uint32_t first_timestamp = header->timestamp;
1759   int extracted_samples = 0;
1760 
1761   // Packet extraction loop.
1762   do {
1763     timestamp_ = header->timestamp;
1764     int discard_count = 0;
1765     Packet* packet = packet_buffer_->GetNextPacket(&discard_count);
1766     // |header| may be invalid after the |packet_buffer_| operation.
1767     header = NULL;
1768     if (!packet) {
1769       LOG_FERR1(LS_ERROR, GetNextPacket, discard_count) <<
1770           "Should always be able to extract a packet here";
1771       assert(false);  // Should always be able to extract a packet here.
1772       return -1;
1773     }
1774     stats_.PacketsDiscarded(discard_count);
1775     // Store waiting time in ms; packets->waiting_time is in "output blocks".
1776     stats_.StoreWaitingTime(packet->waiting_time * kOutputSizeMs);
1777     assert(packet->payload_length > 0);
1778     packet_list->push_back(packet);  // Store packet in list.
1779 
1780     if (first_packet) {
1781       first_packet = false;
1782       decoded_packet_sequence_number_ = prev_sequence_number =
1783           packet->header.sequenceNumber;
1784       decoded_packet_timestamp_ = prev_timestamp = packet->header.timestamp;
1785       prev_payload_type = packet->header.payloadType;
1786     }
1787 
1788     // Store number of extracted samples.
1789     int packet_duration = 0;
1790     AudioDecoder* decoder = decoder_database_->GetDecoder(
1791         packet->header.payloadType);
1792     if (decoder) {
1793       if (packet->sync_packet) {
1794         packet_duration = decoder_frame_length_;
1795       } else {
1796         packet_duration = packet->primary ?
1797             decoder->PacketDuration(packet->payload, packet->payload_length) :
1798             decoder->PacketDurationRedundant(packet->payload,
1799                                              packet->payload_length);
1800       }
1801     } else {
1802       LOG_FERR1(LS_WARNING, GetDecoder, packet->header.payloadType) <<
1803           "Could not find a decoder for a packet about to be extracted.";
1804       assert(false);
1805     }
1806     if (packet_duration <= 0) {
1807       // Decoder did not return a packet duration. Assume that the packet
1808       // contains the same number of samples as the previous one.
1809       packet_duration = decoder_frame_length_;
1810     }
1811     extracted_samples = packet->header.timestamp - first_timestamp +
1812         packet_duration;
1813 
1814     // Check what packet is available next.
1815     header = packet_buffer_->NextRtpHeader();
1816     next_packet_available = false;
1817     if (header && prev_payload_type == header->payloadType) {
1818       int16_t seq_no_diff = header->sequenceNumber - prev_sequence_number;
1819       int32_t ts_diff = header->timestamp - prev_timestamp;
1820       if (seq_no_diff == 1 ||
1821           (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) {
1822         // The next sequence number is available, or the next part of a packet
1823         // that was split into pieces upon insertion.
1824         next_packet_available = true;
1825       }
1826       prev_sequence_number = header->sequenceNumber;
1827     }
1828   } while (extracted_samples < required_samples && next_packet_available);
1829 
1830   return extracted_samples;
1831 }
1832 
UpdatePlcComponents(int fs_hz,size_t channels)1833 void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) {
1834   // Delete objects and create new ones.
1835   expand_.reset(expand_factory_->Create(background_noise_.get(),
1836                                         sync_buffer_.get(), &random_vector_,
1837                                         fs_hz, channels));
1838   merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get()));
1839 }
1840 
SetSampleRateAndChannels(int fs_hz,size_t channels)1841 void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) {
1842   LOG_API2(fs_hz, channels);
1843   // TODO(hlundin): Change to an enumerator and skip assert.
1844   assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz ==  32000 || fs_hz == 48000);
1845   assert(channels > 0);
1846 
1847   fs_hz_ = fs_hz;
1848   fs_mult_ = fs_hz / 8000;
1849   output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
1850   decoder_frame_length_ = 3 * output_size_samples_;  // Initialize to 30ms.
1851 
1852   last_mode_ = kModeNormal;
1853 
1854   // Create a new array of mute factors and set all to 1.
1855   mute_factor_array_.reset(new int16_t[channels]);
1856   for (size_t i = 0; i < channels; ++i) {
1857     mute_factor_array_[i] = 16384;  // 1.0 in Q14.
1858   }
1859 
1860   // Reset comfort noise decoder, if there is one active.
1861   AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1862   if (cng_decoder) {
1863     cng_decoder->Init();
1864   }
1865 
1866   // Reinit post-decode VAD with new sample rate.
1867   assert(vad_.get());  // Cannot be NULL here.
1868   vad_->Init();
1869 
1870   // Delete algorithm buffer and create a new one.
1871   algorithm_buffer_.reset(new AudioMultiVector(channels));
1872 
1873   // Delete sync buffer and create a new one.
1874   sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_));
1875 
1876 
1877   // Delete BackgroundNoise object and create a new one, while preserving its
1878   // mode.
1879   NetEqBackgroundNoiseMode current_mode = kBgnOn;
1880   if (background_noise_.get())
1881     current_mode = background_noise_->mode();
1882   background_noise_.reset(new BackgroundNoise(channels));
1883   background_noise_->set_mode(current_mode);
1884 
1885   // Reset random vector.
1886   random_vector_.Reset();
1887 
1888   UpdatePlcComponents(fs_hz, channels);
1889 
1890   // Move index so that we create a small set of future samples (all 0).
1891   sync_buffer_->set_next_index(sync_buffer_->next_index() -
1892       expand_->overlap_length());
1893 
1894   normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_,
1895                            expand_.get()));
1896   accelerate_.reset(
1897       accelerate_factory_->Create(fs_hz, channels, *background_noise_));
1898   preemptive_expand_.reset(preemptive_expand_factory_->Create(
1899       fs_hz, channels,
1900       *background_noise_,
1901       static_cast<int>(expand_->overlap_length())));
1902 
1903   // Delete ComfortNoise object and create a new one.
1904   comfort_noise_.reset(new ComfortNoise(fs_hz, decoder_database_.get(),
1905                                         sync_buffer_.get()));
1906 
1907   // Verify that |decoded_buffer_| is long enough.
1908   if (decoded_buffer_length_ < kMaxFrameSize * channels) {
1909     // Reallocate to larger size.
1910     decoded_buffer_length_ = kMaxFrameSize * channels;
1911     decoded_buffer_.reset(new int16_t[decoded_buffer_length_]);
1912   }
1913 
1914   // Create DecisionLogic if it is not created yet, then communicate new sample
1915   // rate and output size to DecisionLogic object.
1916   if (!decision_logic_.get()) {
1917     CreateDecisionLogic(kPlayoutOn);
1918   }
1919   decision_logic_->SetSampleRate(fs_hz_, output_size_samples_);
1920 }
1921 
LastOutputType()1922 NetEqOutputType NetEqImpl::LastOutputType() {
1923   assert(vad_.get());
1924   assert(expand_.get());
1925   if (last_mode_ == kModeCodecInternalCng || last_mode_ == kModeRfc3389Cng) {
1926     return kOutputCNG;
1927   } else if (last_mode_ == kModeExpand && expand_->MuteFactor(0) == 0) {
1928     // Expand mode has faded down to background noise only (very long expand).
1929     return kOutputPLCtoCNG;
1930   } else if (last_mode_ == kModeExpand) {
1931     return kOutputPLC;
1932   } else if (vad_->running() && !vad_->active_speech()) {
1933     return kOutputVADPassive;
1934   } else {
1935     return kOutputNormal;
1936   }
1937 }
1938 
CreateDecisionLogic(NetEqPlayoutMode mode)1939 void NetEqImpl::CreateDecisionLogic(NetEqPlayoutMode mode) {
1940   decision_logic_.reset(DecisionLogic::Create(fs_hz_, output_size_samples_,
1941                                               mode,
1942                                               decoder_database_.get(),
1943                                               *packet_buffer_.get(),
1944                                               delay_manager_.get(),
1945                                               buffer_level_filter_.get()));
1946 }
1947 }  // namespace webrtc
1948