1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_coding/main/acm2/acm_receiver.h"
12
13 #include <stdlib.h> // malloc
14
15 #include <algorithm> // sort
16 #include <vector>
17
18 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
19 #include "webrtc/common_types.h"
20 #include "webrtc/modules/audio_coding/main/acm2/acm_common_defs.h"
21 #include "webrtc/modules/audio_coding/main/acm2/acm_resampler.h"
22 #include "webrtc/modules/audio_coding/main/acm2/call_statistics.h"
23 #include "webrtc/modules/audio_coding/main/acm2/nack.h"
24 #include "webrtc/modules/audio_coding/neteq/interface/audio_decoder.h"
25 #include "webrtc/modules/audio_coding/neteq/interface/neteq.h"
26 #include "webrtc/system_wrappers/interface/clock.h"
27 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
28 #include "webrtc/system_wrappers/interface/logging.h"
29 #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h"
30 #include "webrtc/system_wrappers/interface/tick_util.h"
31 #include "webrtc/system_wrappers/interface/trace.h"
32
33 namespace webrtc {
34
35 namespace acm2 {
36
37 namespace {
38
39 const int kNackThresholdPackets = 2;
40
41 // |vad_activity_| field of |audio_frame| is set to |previous_audio_activity_|
42 // before the call to this function.
SetAudioFrameActivityAndType(bool vad_enabled,NetEqOutputType type,AudioFrame * audio_frame)43 void SetAudioFrameActivityAndType(bool vad_enabled,
44 NetEqOutputType type,
45 AudioFrame* audio_frame) {
46 if (vad_enabled) {
47 switch (type) {
48 case kOutputNormal: {
49 audio_frame->vad_activity_ = AudioFrame::kVadActive;
50 audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
51 break;
52 }
53 case kOutputVADPassive: {
54 audio_frame->vad_activity_ = AudioFrame::kVadPassive;
55 audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
56 break;
57 }
58 case kOutputCNG: {
59 audio_frame->vad_activity_ = AudioFrame::kVadPassive;
60 audio_frame->speech_type_ = AudioFrame::kCNG;
61 break;
62 }
63 case kOutputPLC: {
64 // Don't change |audio_frame->vad_activity_|, it should be the same as
65 // |previous_audio_activity_|.
66 audio_frame->speech_type_ = AudioFrame::kPLC;
67 break;
68 }
69 case kOutputPLCtoCNG: {
70 audio_frame->vad_activity_ = AudioFrame::kVadPassive;
71 audio_frame->speech_type_ = AudioFrame::kPLCCNG;
72 break;
73 }
74 default:
75 assert(false);
76 }
77 } else {
78 // Always return kVadUnknown when receive VAD is inactive
79 audio_frame->vad_activity_ = AudioFrame::kVadUnknown;
80 switch (type) {
81 case kOutputNormal: {
82 audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
83 break;
84 }
85 case kOutputCNG: {
86 audio_frame->speech_type_ = AudioFrame::kCNG;
87 break;
88 }
89 case kOutputPLC: {
90 audio_frame->speech_type_ = AudioFrame::kPLC;
91 break;
92 }
93 case kOutputPLCtoCNG: {
94 audio_frame->speech_type_ = AudioFrame::kPLCCNG;
95 break;
96 }
97 case kOutputVADPassive: {
98 // Normally, we should no get any VAD decision if post-decoding VAD is
99 // not active. However, if post-decoding VAD has been active then
100 // disabled, we might be here for couple of frames.
101 audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
102 LOG_F(LS_WARNING) << "Post-decoding VAD is disabled but output is "
103 << "labeled VAD-passive";
104 break;
105 }
106 default:
107 assert(false);
108 }
109 }
110 }
111
112 // Is the given codec a CNG codec?
IsCng(int codec_id)113 bool IsCng(int codec_id) {
114 return (codec_id == ACMCodecDB::kCNNB || codec_id == ACMCodecDB::kCNWB ||
115 codec_id == ACMCodecDB::kCNSWB || codec_id == ACMCodecDB::kCNFB);
116 }
117
118 } // namespace
119
AcmReceiver(const AudioCodingModule::Config & config)120 AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config)
121 : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
122 id_(config.id),
123 last_audio_decoder_(-1), // Invalid value.
124 previous_audio_activity_(AudioFrame::kVadPassive),
125 current_sample_rate_hz_(config.neteq_config.sample_rate_hz),
126 nack_(),
127 nack_enabled_(false),
128 neteq_(NetEq::Create(config.neteq_config)),
129 decode_lock_(RWLockWrapper::CreateRWLock()),
130 vad_enabled_(true),
131 clock_(config.clock),
132 av_sync_(false),
133 initial_delay_manager_(),
134 missing_packets_sync_stream_(),
135 late_packets_sync_stream_() {
136 assert(clock_);
137 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) {
138 decoders_[n].registered = false;
139 }
140
141 // Make sure we are on the same page as NetEq. Post-decode VAD is disabled by
142 // default in NetEq4, however, Audio Conference Mixer relies on VAD decision
143 // and fails if VAD decision is not provided.
144 if (vad_enabled_)
145 neteq_->EnableVad();
146 else
147 neteq_->DisableVad();
148 }
149
~AcmReceiver()150 AcmReceiver::~AcmReceiver() {
151 delete neteq_;
152 delete decode_lock_;
153 }
154
SetMinimumDelay(int delay_ms)155 int AcmReceiver::SetMinimumDelay(int delay_ms) {
156 if (neteq_->SetMinimumDelay(delay_ms))
157 return 0;
158 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms);
159 return -1;
160 }
161
SetInitialDelay(int delay_ms)162 int AcmReceiver::SetInitialDelay(int delay_ms) {
163 if (delay_ms < 0 || delay_ms > 10000) {
164 return -1;
165 }
166 CriticalSectionScoped lock(crit_sect_.get());
167
168 if (delay_ms == 0) {
169 av_sync_ = false;
170 initial_delay_manager_.reset();
171 missing_packets_sync_stream_.reset();
172 late_packets_sync_stream_.reset();
173 neteq_->SetMinimumDelay(0);
174 return 0;
175 }
176
177 if (av_sync_ && initial_delay_manager_->PacketBuffered()) {
178 // Too late for this API. Only works before a call is started.
179 return -1;
180 }
181
182 // Most of places NetEq calls are not within AcmReceiver's critical section to
183 // improve performance. Here, this call has to be placed before the following
184 // block, therefore, we keep it inside critical section. Otherwise, we have to
185 // release |neteq_crit_sect_| and acquire it again, which seems an overkill.
186 if (!neteq_->SetMinimumDelay(delay_ms))
187 return -1;
188
189 const int kLatePacketThreshold = 5;
190 av_sync_ = true;
191 initial_delay_manager_.reset(new InitialDelayManager(delay_ms,
192 kLatePacketThreshold));
193 missing_packets_sync_stream_.reset(new InitialDelayManager::SyncStream);
194 late_packets_sync_stream_.reset(new InitialDelayManager::SyncStream);
195 return 0;
196 }
197
SetMaximumDelay(int delay_ms)198 int AcmReceiver::SetMaximumDelay(int delay_ms) {
199 if (neteq_->SetMaximumDelay(delay_ms))
200 return 0;
201 LOG_FERR1(LS_ERROR, "AcmReceiver::SetExtraDelay", delay_ms);
202 return -1;
203 }
204
LeastRequiredDelayMs() const205 int AcmReceiver::LeastRequiredDelayMs() const {
206 return neteq_->LeastRequiredDelayMs();
207 }
208
current_sample_rate_hz() const209 int AcmReceiver::current_sample_rate_hz() const {
210 CriticalSectionScoped lock(crit_sect_.get());
211 return current_sample_rate_hz_;
212 }
213
214 // TODO(turajs): use one set of enumerators, e.g. the one defined in
215 // common_types.h
SetPlayoutMode(AudioPlayoutMode mode)216 void AcmReceiver::SetPlayoutMode(AudioPlayoutMode mode) {
217 enum NetEqPlayoutMode playout_mode = kPlayoutOn;
218 enum NetEqBackgroundNoiseMode bgn_mode = kBgnOn;
219 switch (mode) {
220 case voice:
221 playout_mode = kPlayoutOn;
222 bgn_mode = kBgnOn;
223 break;
224 case fax: // No change to background noise mode.
225 playout_mode = kPlayoutFax;
226 bgn_mode = neteq_->BackgroundNoiseMode();
227 break;
228 case streaming:
229 playout_mode = kPlayoutStreaming;
230 bgn_mode = kBgnOff;
231 break;
232 case off:
233 playout_mode = kPlayoutOff;
234 bgn_mode = kBgnOff;
235 break;
236 }
237 neteq_->SetPlayoutMode(playout_mode);
238 neteq_->SetBackgroundNoiseMode(bgn_mode);
239 }
240
PlayoutMode() const241 AudioPlayoutMode AcmReceiver::PlayoutMode() const {
242 AudioPlayoutMode acm_mode = voice;
243 NetEqPlayoutMode mode = neteq_->PlayoutMode();
244 switch (mode) {
245 case kPlayoutOn:
246 acm_mode = voice;
247 break;
248 case kPlayoutOff:
249 acm_mode = off;
250 break;
251 case kPlayoutFax:
252 acm_mode = fax;
253 break;
254 case kPlayoutStreaming:
255 acm_mode = streaming;
256 break;
257 default:
258 assert(false);
259 }
260 return acm_mode;
261 }
262
InsertPacket(const WebRtcRTPHeader & rtp_header,const uint8_t * incoming_payload,int length_payload)263 int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header,
264 const uint8_t* incoming_payload,
265 int length_payload) {
266 uint32_t receive_timestamp = 0;
267 InitialDelayManager::PacketType packet_type =
268 InitialDelayManager::kUndefinedPacket;
269 bool new_codec = false;
270 const RTPHeader* header = &rtp_header.header; // Just a shorthand.
271
272 {
273 CriticalSectionScoped lock(crit_sect_.get());
274
275 int codec_id = RtpHeaderToCodecIndex(*header, incoming_payload);
276 if (codec_id < 0) {
277 LOG_F(LS_ERROR) << "Payload-type " << header->payloadType
278 << " is not registered.";
279 return -1;
280 }
281 assert(codec_id < ACMCodecDB::kMaxNumCodecs);
282 const int sample_rate_hz = ACMCodecDB::CodecFreq(codec_id);
283 receive_timestamp = NowInTimestamp(sample_rate_hz);
284
285 if (IsCng(codec_id)) {
286 // If this is a CNG while the audio codec is not mono skip pushing in
287 // packets into NetEq.
288 if (last_audio_decoder_ >= 0 &&
289 decoders_[last_audio_decoder_].channels > 1)
290 return 0;
291 packet_type = InitialDelayManager::kCngPacket;
292 } else if (codec_id == ACMCodecDB::kAVT) {
293 packet_type = InitialDelayManager::kAvtPacket;
294 } else {
295 if (codec_id != last_audio_decoder_) {
296 // This is either the first audio packet or send codec is changed.
297 // Therefore, either NetEq buffer is empty or will be flushed when this
298 // packet inserted. Note that |last_audio_decoder_| is initialized to
299 // an invalid value (-1), hence, the above condition is true for the
300 // very first audio packet.
301 new_codec = true;
302
303 // Updating NACK'sampling rate is required, either first packet is
304 // received or codec is changed. Furthermore, reset is required if codec
305 // is changed (NetEq flushes its buffer so NACK should reset its list).
306 if (nack_enabled_) {
307 assert(nack_.get());
308 nack_->Reset();
309 nack_->UpdateSampleRate(sample_rate_hz);
310 }
311 last_audio_decoder_ = codec_id;
312 }
313 packet_type = InitialDelayManager::kAudioPacket;
314 }
315
316 if (nack_enabled_) {
317 assert(nack_.get());
318 nack_->UpdateLastReceivedPacket(header->sequenceNumber,
319 header->timestamp);
320 }
321
322 if (av_sync_) {
323 assert(initial_delay_manager_.get());
324 assert(missing_packets_sync_stream_.get());
325 // This updates |initial_delay_manager_| and specifies an stream of
326 // sync-packets, if required to be inserted. We insert the sync-packets
327 // when AcmReceiver lock is released and |decoder_lock_| is acquired.
328 initial_delay_manager_->UpdateLastReceivedPacket(
329 rtp_header, receive_timestamp, packet_type, new_codec, sample_rate_hz,
330 missing_packets_sync_stream_.get());
331 }
332 } // |crit_sect_| is released.
333
334 {
335 WriteLockScoped lock_codecs(*decode_lock_); // Lock to prevent an encoding.
336
337 // If |missing_packets_sync_stream_| is allocated then we are in AV-sync and
338 // we may need to insert sync-packets. We don't check |av_sync_| as we are
339 // outside AcmReceiver's critical section.
340 if (missing_packets_sync_stream_.get()) {
341 InsertStreamOfSyncPackets(missing_packets_sync_stream_.get());
342 }
343
344 if (neteq_->InsertPacket(rtp_header, incoming_payload, length_payload,
345 receive_timestamp) < 0) {
346 LOG_FERR1(LS_ERROR, "AcmReceiver::InsertPacket", header->payloadType) <<
347 " Failed to insert packet";
348 return -1;
349 }
350 }
351 return 0;
352 }
353
GetAudio(int desired_freq_hz,AudioFrame * audio_frame)354 int AcmReceiver::GetAudio(int desired_freq_hz, AudioFrame* audio_frame) {
355 enum NetEqOutputType type;
356 int16_t* ptr_audio_buffer = audio_frame->data_;
357 int samples_per_channel;
358 int num_channels;
359 bool return_silence = false;
360
361 {
362 // Accessing members, take the lock.
363 CriticalSectionScoped lock(crit_sect_.get());
364
365 if (av_sync_) {
366 assert(initial_delay_manager_.get());
367 assert(late_packets_sync_stream_.get());
368 return_silence = GetSilence(desired_freq_hz, audio_frame);
369 uint32_t timestamp_now = NowInTimestamp(current_sample_rate_hz_);
370 initial_delay_manager_->LatePackets(timestamp_now,
371 late_packets_sync_stream_.get());
372 }
373
374 if (!return_silence) {
375 // This is our initial guess regarding whether a resampling will be
376 // required. It is based on previous sample rate of netEq. Most often,
377 // this is a correct guess, however, in case that incoming payload changes
378 // the resampling might might be needed. By doing so, we avoid an
379 // unnecessary memcpy().
380 if (desired_freq_hz != -1 &&
381 current_sample_rate_hz_ != desired_freq_hz) {
382 ptr_audio_buffer = audio_buffer_;
383 }
384 }
385 }
386
387 {
388 WriteLockScoped lock_codecs(*decode_lock_); // Lock to prevent an encoding.
389
390 // If |late_packets_sync_stream_| is allocated then we have been in AV-sync
391 // mode and we might have to insert sync-packets.
392 if (late_packets_sync_stream_.get()) {
393 InsertStreamOfSyncPackets(late_packets_sync_stream_.get());
394 if (return_silence) // Silence generated, don't pull from NetEq.
395 return 0;
396 }
397
398 if (neteq_->GetAudio(AudioFrame::kMaxDataSizeSamples,
399 ptr_audio_buffer,
400 &samples_per_channel,
401 &num_channels, &type) != NetEq::kOK) {
402 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "NetEq Failed.";
403 return -1;
404 }
405 }
406
407 // Accessing members, take the lock.
408 CriticalSectionScoped lock(crit_sect_.get());
409
410 // Update NACK.
411 int decoded_sequence_num = 0;
412 uint32_t decoded_timestamp = 0;
413 bool update_nack = nack_enabled_ && // Update NACK only if it is enabled.
414 neteq_->DecodedRtpInfo(&decoded_sequence_num, &decoded_timestamp);
415 if (update_nack) {
416 assert(nack_.get());
417 nack_->UpdateLastDecodedPacket(decoded_sequence_num, decoded_timestamp);
418 }
419
420 // NetEq always returns 10 ms of audio.
421 current_sample_rate_hz_ = samples_per_channel * 100;
422
423 // Update if resampling is required.
424 bool need_resampling = (desired_freq_hz != -1) &&
425 (current_sample_rate_hz_ != desired_freq_hz);
426
427 if (ptr_audio_buffer == audio_buffer_) {
428 // Data is written to local buffer.
429 if (need_resampling) {
430 samples_per_channel =
431 resampler_.Resample10Msec(audio_buffer_,
432 current_sample_rate_hz_,
433 desired_freq_hz,
434 num_channels,
435 AudioFrame::kMaxDataSizeSamples,
436 audio_frame->data_);
437 if (samples_per_channel < 0) {
438 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed.";
439 return -1;
440 }
441 } else {
442 // We might end up here ONLY if codec is changed.
443 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel *
444 num_channels * sizeof(int16_t));
445 }
446 } else {
447 // Data is written into |audio_frame|.
448 if (need_resampling) {
449 // We might end up here ONLY if codec is changed.
450 samples_per_channel =
451 resampler_.Resample10Msec(audio_frame->data_,
452 current_sample_rate_hz_,
453 desired_freq_hz,
454 num_channels,
455 AudioFrame::kMaxDataSizeSamples,
456 audio_buffer_);
457 if (samples_per_channel < 0) {
458 LOG_FERR0(LS_ERROR, "AcmReceiver::GetAudio") << "Resampler Failed.";
459 return -1;
460 }
461 memcpy(audio_frame->data_, audio_buffer_, samples_per_channel *
462 num_channels * sizeof(int16_t));
463 }
464 }
465
466 audio_frame->num_channels_ = num_channels;
467 audio_frame->samples_per_channel_ = samples_per_channel;
468 audio_frame->sample_rate_hz_ = samples_per_channel * 100;
469
470 // Should set |vad_activity| before calling SetAudioFrameActivityAndType().
471 audio_frame->vad_activity_ = previous_audio_activity_;
472 SetAudioFrameActivityAndType(vad_enabled_, type, audio_frame);
473 previous_audio_activity_ = audio_frame->vad_activity_;
474 call_stats_.DecodedByNetEq(audio_frame->speech_type_);
475
476 // Computes the RTP timestamp of the first sample in |audio_frame| from
477 // |GetPlayoutTimestamp|, which is the timestamp of the last sample of
478 // |audio_frame|.
479 uint32_t playout_timestamp = 0;
480 if (GetPlayoutTimestamp(&playout_timestamp)) {
481 audio_frame->timestamp_ =
482 playout_timestamp - audio_frame->samples_per_channel_;
483 } else {
484 // Remain 0 until we have a valid |playout_timestamp|.
485 audio_frame->timestamp_ = 0;
486 }
487
488 return 0;
489 }
490
AddCodec(int acm_codec_id,uint8_t payload_type,int channels,AudioDecoder * audio_decoder)491 int32_t AcmReceiver::AddCodec(int acm_codec_id,
492 uint8_t payload_type,
493 int channels,
494 AudioDecoder* audio_decoder) {
495 assert(acm_codec_id >= 0 && acm_codec_id < ACMCodecDB::kMaxNumCodecs);
496 NetEqDecoder neteq_decoder = ACMCodecDB::neteq_decoders_[acm_codec_id];
497
498 // Make sure the right decoder is registered for Opus.
499 if (neteq_decoder == kDecoderOpus && channels == 2) {
500 neteq_decoder = kDecoderOpus_2ch;
501 }
502
503 CriticalSectionScoped lock(crit_sect_.get());
504
505 // The corresponding NetEq decoder ID.
506 // If this coder has been registered before.
507 if (decoders_[acm_codec_id].registered) {
508 if (decoders_[acm_codec_id].payload_type == payload_type &&
509 decoders_[acm_codec_id].channels == channels) {
510 // Re-registering the same codec with the same payload-type. Do nothing
511 // and return.
512 return 0;
513 }
514
515 // Changing the payload-type or number of channels for this codec.
516 // First unregister. Then register with new payload-type/channels.
517 if (neteq_->RemovePayloadType(decoders_[acm_codec_id].payload_type) !=
518 NetEq::kOK) {
519 LOG_F(LS_ERROR) << "Cannot remover payload "
520 << decoders_[acm_codec_id].payload_type;
521 return -1;
522 }
523 }
524
525 int ret_val;
526 if (!audio_decoder) {
527 ret_val = neteq_->RegisterPayloadType(neteq_decoder, payload_type);
528 } else {
529 ret_val = neteq_->RegisterExternalDecoder(
530 audio_decoder, neteq_decoder, payload_type);
531 }
532 if (ret_val != NetEq::kOK) {
533 LOG_FERR3(LS_ERROR, "AcmReceiver::AddCodec", acm_codec_id, payload_type,
534 channels);
535 // Registration failed, delete the allocated space and set the pointer to
536 // NULL, for the record.
537 decoders_[acm_codec_id].registered = false;
538 return -1;
539 }
540
541 decoders_[acm_codec_id].registered = true;
542 decoders_[acm_codec_id].payload_type = payload_type;
543 decoders_[acm_codec_id].channels = channels;
544 return 0;
545 }
546
EnableVad()547 void AcmReceiver::EnableVad() {
548 neteq_->EnableVad();
549 CriticalSectionScoped lock(crit_sect_.get());
550 vad_enabled_ = true;
551 }
552
DisableVad()553 void AcmReceiver::DisableVad() {
554 neteq_->DisableVad();
555 CriticalSectionScoped lock(crit_sect_.get());
556 vad_enabled_ = false;
557 }
558
FlushBuffers()559 void AcmReceiver::FlushBuffers() {
560 neteq_->FlushBuffers();
561 }
562
563 // If failed in removing one of the codecs, this method continues to remove as
564 // many as it can.
RemoveAllCodecs()565 int AcmReceiver::RemoveAllCodecs() {
566 int ret_val = 0;
567 CriticalSectionScoped lock(crit_sect_.get());
568 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) {
569 if (decoders_[n].registered) {
570 if (neteq_->RemovePayloadType(decoders_[n].payload_type) == 0) {
571 decoders_[n].registered = false;
572 } else {
573 LOG_F(LS_ERROR) << "Cannot remove payload "
574 << decoders_[n].payload_type;
575 ret_val = -1;
576 }
577 }
578 }
579 // No codec is registered, invalidate last audio decoder.
580 last_audio_decoder_ = -1;
581 return ret_val;
582 }
583
RemoveCodec(uint8_t payload_type)584 int AcmReceiver::RemoveCodec(uint8_t payload_type) {
585 int codec_index = PayloadType2CodecIndex(payload_type);
586 if (codec_index < 0) { // Such a payload-type is not registered.
587 return 0;
588 }
589 if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK) {
590 LOG_FERR1(LS_ERROR, "AcmReceiver::RemoveCodec", payload_type);
591 return -1;
592 }
593 CriticalSectionScoped lock(crit_sect_.get());
594 decoders_[codec_index].registered = false;
595 if (last_audio_decoder_ == codec_index)
596 last_audio_decoder_ = -1; // Codec is removed, invalidate last decoder.
597 return 0;
598 }
599
set_id(int id)600 void AcmReceiver::set_id(int id) {
601 CriticalSectionScoped lock(crit_sect_.get());
602 id_ = id;
603 }
604
GetPlayoutTimestamp(uint32_t * timestamp)605 bool AcmReceiver::GetPlayoutTimestamp(uint32_t* timestamp) {
606 if (av_sync_) {
607 assert(initial_delay_manager_.get());
608 if (initial_delay_manager_->buffering()) {
609 return initial_delay_manager_->GetPlayoutTimestamp(timestamp);
610 }
611 }
612 return neteq_->GetPlayoutTimestamp(timestamp);
613 }
614
last_audio_codec_id() const615 int AcmReceiver::last_audio_codec_id() const {
616 CriticalSectionScoped lock(crit_sect_.get());
617 return last_audio_decoder_;
618 }
619
last_audio_payload_type() const620 int AcmReceiver::last_audio_payload_type() const {
621 CriticalSectionScoped lock(crit_sect_.get());
622 if (last_audio_decoder_ < 0)
623 return -1;
624 assert(decoders_[last_audio_decoder_].registered);
625 return decoders_[last_audio_decoder_].payload_type;
626 }
627
RedPayloadType() const628 int AcmReceiver::RedPayloadType() const {
629 CriticalSectionScoped lock(crit_sect_.get());
630 if (ACMCodecDB::kRED < 0 ||
631 !decoders_[ACMCodecDB::kRED].registered) {
632 LOG_F(LS_WARNING) << "RED is not registered.";
633 return -1;
634 }
635 return decoders_[ACMCodecDB::kRED].payload_type;
636 }
637
LastAudioCodec(CodecInst * codec) const638 int AcmReceiver::LastAudioCodec(CodecInst* codec) const {
639 CriticalSectionScoped lock(crit_sect_.get());
640 if (last_audio_decoder_ < 0) {
641 return -1;
642 }
643 assert(decoders_[last_audio_decoder_].registered);
644 memcpy(codec, &ACMCodecDB::database_[last_audio_decoder_], sizeof(CodecInst));
645 codec->pltype = decoders_[last_audio_decoder_].payload_type;
646 codec->channels = decoders_[last_audio_decoder_].channels;
647 return 0;
648 }
649
NetworkStatistics(ACMNetworkStatistics * acm_stat)650 void AcmReceiver::NetworkStatistics(ACMNetworkStatistics* acm_stat) {
651 NetEqNetworkStatistics neteq_stat;
652 // NetEq function always returns zero, so we don't check the return value.
653 neteq_->NetworkStatistics(&neteq_stat);
654
655 acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms;
656 acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms;
657 acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false;
658 acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate;
659 acm_stat->currentDiscardRate = neteq_stat.packet_discard_rate;
660 acm_stat->currentExpandRate = neteq_stat.expand_rate;
661 acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate;
662 acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate;
663 acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm;
664 acm_stat->addedSamples = neteq_stat.added_zero_samples;
665
666 std::vector<int> waiting_times;
667 neteq_->WaitingTimes(&waiting_times);
668 size_t size = waiting_times.size();
669 if (size == 0) {
670 acm_stat->meanWaitingTimeMs = -1;
671 acm_stat->medianWaitingTimeMs = -1;
672 acm_stat->minWaitingTimeMs = -1;
673 acm_stat->maxWaitingTimeMs = -1;
674 } else {
675 std::sort(waiting_times.begin(), waiting_times.end());
676 if ((size & 0x1) == 0) {
677 acm_stat->medianWaitingTimeMs = (waiting_times[size / 2 - 1] +
678 waiting_times[size / 2]) / 2;
679 } else {
680 acm_stat->medianWaitingTimeMs = waiting_times[size / 2];
681 }
682 acm_stat->minWaitingTimeMs = waiting_times.front();
683 acm_stat->maxWaitingTimeMs = waiting_times.back();
684 double sum = 0;
685 for (size_t i = 0; i < size; ++i) {
686 sum += waiting_times[i];
687 }
688 acm_stat->meanWaitingTimeMs = static_cast<int>(sum / size);
689 }
690 }
691
DecoderByPayloadType(uint8_t payload_type,CodecInst * codec) const692 int AcmReceiver::DecoderByPayloadType(uint8_t payload_type,
693 CodecInst* codec) const {
694 CriticalSectionScoped lock(crit_sect_.get());
695 int codec_index = PayloadType2CodecIndex(payload_type);
696 if (codec_index < 0) {
697 LOG_FERR1(LS_ERROR, "AcmReceiver::DecoderByPayloadType", payload_type);
698 return -1;
699 }
700 memcpy(codec, &ACMCodecDB::database_[codec_index], sizeof(CodecInst));
701 codec->pltype = decoders_[codec_index].payload_type;
702 codec->channels = decoders_[codec_index].channels;
703 return 0;
704 }
705
PayloadType2CodecIndex(uint8_t payload_type) const706 int AcmReceiver::PayloadType2CodecIndex(uint8_t payload_type) const {
707 for (int n = 0; n < ACMCodecDB::kMaxNumCodecs; ++n) {
708 if (decoders_[n].registered && decoders_[n].payload_type == payload_type) {
709 return n;
710 }
711 }
712 return -1;
713 }
714
EnableNack(size_t max_nack_list_size)715 int AcmReceiver::EnableNack(size_t max_nack_list_size) {
716 // Don't do anything if |max_nack_list_size| is out of range.
717 if (max_nack_list_size == 0 || max_nack_list_size > Nack::kNackListSizeLimit)
718 return -1;
719
720 CriticalSectionScoped lock(crit_sect_.get());
721 if (!nack_enabled_) {
722 nack_.reset(Nack::Create(kNackThresholdPackets));
723 nack_enabled_ = true;
724
725 // Sampling rate might need to be updated if we change from disable to
726 // enable. Do it if the receive codec is valid.
727 if (last_audio_decoder_ >= 0) {
728 nack_->UpdateSampleRate(
729 ACMCodecDB::database_[last_audio_decoder_].plfreq);
730 }
731 }
732 return nack_->SetMaxNackListSize(max_nack_list_size);
733 }
734
DisableNack()735 void AcmReceiver::DisableNack() {
736 CriticalSectionScoped lock(crit_sect_.get());
737 nack_.reset(); // Memory is released.
738 nack_enabled_ = false;
739 }
740
GetNackList(int round_trip_time_ms) const741 std::vector<uint16_t> AcmReceiver::GetNackList(
742 int round_trip_time_ms) const {
743 CriticalSectionScoped lock(crit_sect_.get());
744 if (round_trip_time_ms < 0) {
745 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
746 "GetNackList: round trip time cannot be negative."
747 " round_trip_time_ms=%d", round_trip_time_ms);
748 }
749 if (nack_enabled_ && round_trip_time_ms >= 0) {
750 assert(nack_.get());
751 return nack_->GetNackList(round_trip_time_ms);
752 }
753 std::vector<uint16_t> empty_list;
754 return empty_list;
755 }
756
ResetInitialDelay()757 void AcmReceiver::ResetInitialDelay() {
758 {
759 CriticalSectionScoped lock(crit_sect_.get());
760 av_sync_ = false;
761 initial_delay_manager_.reset(NULL);
762 missing_packets_sync_stream_.reset(NULL);
763 late_packets_sync_stream_.reset(NULL);
764 }
765 neteq_->SetMinimumDelay(0);
766 // TODO(turajs): Should NetEq Buffer be flushed?
767 }
768
769 // This function is called within critical section, no need to acquire a lock.
GetSilence(int desired_sample_rate_hz,AudioFrame * frame)770 bool AcmReceiver::GetSilence(int desired_sample_rate_hz, AudioFrame* frame) {
771 assert(av_sync_);
772 assert(initial_delay_manager_.get());
773 if (!initial_delay_manager_->buffering()) {
774 return false;
775 }
776
777 // We stop accumulating packets, if the number of packets or the total size
778 // exceeds a threshold.
779 int num_packets;
780 int max_num_packets;
781 const float kBufferingThresholdScale = 0.9f;
782 neteq_->PacketBufferStatistics(&num_packets, &max_num_packets);
783 if (num_packets > max_num_packets * kBufferingThresholdScale) {
784 initial_delay_manager_->DisableBuffering();
785 return false;
786 }
787
788 // Update statistics.
789 call_stats_.DecodedBySilenceGenerator();
790
791 // Set the values if already got a packet, otherwise set to default values.
792 if (last_audio_decoder_ >= 0) {
793 current_sample_rate_hz_ = ACMCodecDB::database_[last_audio_decoder_].plfreq;
794 frame->num_channels_ = decoders_[last_audio_decoder_].channels;
795 } else {
796 frame->num_channels_ = 1;
797 }
798
799 // Set the audio frame's sampling frequency.
800 if (desired_sample_rate_hz > 0) {
801 frame->sample_rate_hz_ = desired_sample_rate_hz;
802 } else {
803 frame->sample_rate_hz_ = current_sample_rate_hz_;
804 }
805
806 frame->samples_per_channel_ = frame->sample_rate_hz_ / 100; // Always 10 ms.
807 frame->speech_type_ = AudioFrame::kCNG;
808 frame->vad_activity_ = AudioFrame::kVadPassive;
809 int samples = frame->samples_per_channel_ * frame->num_channels_;
810 memset(frame->data_, 0, samples * sizeof(int16_t));
811 return true;
812 }
813
BackgroundNoiseModeForTest() const814 NetEqBackgroundNoiseMode AcmReceiver::BackgroundNoiseModeForTest() const {
815 return neteq_->BackgroundNoiseMode();
816 }
817
RtpHeaderToCodecIndex(const RTPHeader & rtp_header,const uint8_t * payload) const818 int AcmReceiver::RtpHeaderToCodecIndex(
819 const RTPHeader &rtp_header, const uint8_t* payload) const {
820 uint8_t payload_type = rtp_header.payloadType;
821 if (ACMCodecDB::kRED >= 0 && // This ensures that RED is defined in WebRTC.
822 decoders_[ACMCodecDB::kRED].registered &&
823 payload_type == decoders_[ACMCodecDB::kRED].payload_type) {
824 // This is a RED packet, get the payload of the audio codec.
825 payload_type = payload[0] & 0x7F;
826 }
827
828 // Check if the payload is registered.
829 return PayloadType2CodecIndex(payload_type);
830 }
831
NowInTimestamp(int decoder_sampling_rate) const832 uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const {
833 // Down-cast the time to (32-6)-bit since we only care about
834 // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
835 // We masked 6 most significant bits of 32-bit so there is no overflow in
836 // the conversion from milliseconds to timestamp.
837 const uint32_t now_in_ms = static_cast<uint32_t>(
838 clock_->TimeInMilliseconds() & 0x03ffffff);
839 return static_cast<uint32_t>(
840 (decoder_sampling_rate / 1000) * now_in_ms);
841 }
842
843 // This function only interacts with |neteq_|, therefore, it does not have to
844 // be within critical section of AcmReceiver. It is inserting packets
845 // into NetEq, so we call it when |decode_lock_| is acquired. However, this is
846 // not essential as sync-packets do not interact with codecs (especially BWE).
InsertStreamOfSyncPackets(InitialDelayManager::SyncStream * sync_stream)847 void AcmReceiver::InsertStreamOfSyncPackets(
848 InitialDelayManager::SyncStream* sync_stream) {
849 assert(sync_stream);
850 assert(av_sync_);
851 for (int n = 0; n < sync_stream->num_sync_packets; ++n) {
852 neteq_->InsertSyncPacket(sync_stream->rtp_info,
853 sync_stream->receive_timestamp);
854 ++sync_stream->rtp_info.header.sequenceNumber;
855 sync_stream->rtp_info.header.timestamp += sync_stream->timestamp_step;
856 sync_stream->receive_timestamp += sync_stream->timestamp_step;
857 }
858 }
859
GetDecodingCallStatistics(AudioDecodingCallStats * stats) const860 void AcmReceiver::GetDecodingCallStatistics(
861 AudioDecodingCallStats* stats) const {
862 CriticalSectionScoped lock(crit_sect_.get());
863 *stats = call_stats_.GetDecodingStatistics();
864 }
865
866 } // namespace acm2
867
868 } // namespace webrtc
869