1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "audio/voip/audio_ingress.h"
12
13 #include <algorithm>
14 #include <utility>
15 #include <vector>
16
17 #include "api/audio_codecs/audio_format.h"
18 #include "audio/utility/audio_frame_operations.h"
19 #include "modules/audio_coding/include/audio_coding_module.h"
20 #include "rtc_base/logging.h"
21 #include "rtc_base/numerics/safe_minmax.h"
22
23 namespace webrtc {
24
25 namespace {
26
CreateAcmConfig(rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)27 AudioCodingModule::Config CreateAcmConfig(
28 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) {
29 AudioCodingModule::Config acm_config;
30 acm_config.neteq_config.enable_muted_state = true;
31 acm_config.decoder_factory = decoder_factory;
32 return acm_config;
33 }
34
35 } // namespace
36
AudioIngress(RtpRtcpInterface * rtp_rtcp,Clock * clock,ReceiveStatistics * receive_statistics,rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)37 AudioIngress::AudioIngress(
38 RtpRtcpInterface* rtp_rtcp,
39 Clock* clock,
40 ReceiveStatistics* receive_statistics,
41 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
42 : playing_(false),
43 remote_ssrc_(0),
44 first_rtp_timestamp_(-1),
45 rtp_receive_statistics_(receive_statistics),
46 rtp_rtcp_(rtp_rtcp),
47 acm_receiver_(CreateAcmConfig(decoder_factory)),
48 ntp_estimator_(clock) {}
49
50 AudioIngress::~AudioIngress() = default;
51
GetAudioFrameWithInfo(int sampling_rate,AudioFrame * audio_frame)52 AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo(
53 int sampling_rate,
54 AudioFrame* audio_frame) {
55 audio_frame->sample_rate_hz_ = sampling_rate;
56
57 // Get 10ms raw PCM data from the ACM.
58 bool muted = false;
59 if (acm_receiver_.GetAudio(sampling_rate, audio_frame, &muted) == -1) {
60 RTC_DLOG(LS_ERROR) << "GetAudio() failed!";
61 // In all likelihood, the audio in this frame is garbage. We return an
62 // error so that the audio mixer module doesn't add it to the mix. As
63 // a result, it won't be played out and the actions skipped here are
64 // irrelevant.
65 return AudioMixer::Source::AudioFrameInfo::kError;
66 }
67
68 if (muted) {
69 AudioFrameOperations::Mute(audio_frame);
70 }
71
72 // Measure audio level.
73 constexpr double kAudioSampleDurationSeconds = 0.01;
74 output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
75
76 // Set first rtp timestamp with first audio frame with valid timestamp.
77 if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) {
78 first_rtp_timestamp_ = audio_frame->timestamp_;
79 }
80
81 if (first_rtp_timestamp_ >= 0) {
82 // Compute elapsed and NTP times.
83 int64_t unwrap_timestamp;
84 {
85 MutexLock lock(&lock_);
86 unwrap_timestamp =
87 timestamp_wrap_handler_.Unwrap(audio_frame->timestamp_);
88 audio_frame->ntp_time_ms_ =
89 ntp_estimator_.Estimate(audio_frame->timestamp_);
90 }
91 // For clock rate, default to the playout sampling rate if we haven't
92 // received any packets yet.
93 absl::optional<std::pair<int, SdpAudioFormat>> decoder =
94 acm_receiver_.LastDecoder();
95 int clock_rate = decoder ? decoder->second.clockrate_hz
96 : acm_receiver_.last_output_sample_rate_hz();
97 RTC_DCHECK_GT(clock_rate, 0);
98 audio_frame->elapsed_time_ms_ =
99 (unwrap_timestamp - first_rtp_timestamp_) / (clock_rate / 1000);
100 }
101
102 return muted ? AudioMixer::Source::AudioFrameInfo::kMuted
103 : AudioMixer::Source::AudioFrameInfo::kNormal;
104 }
105
SetReceiveCodecs(const std::map<int,SdpAudioFormat> & codecs)106 void AudioIngress::SetReceiveCodecs(
107 const std::map<int, SdpAudioFormat>& codecs) {
108 {
109 MutexLock lock(&lock_);
110 for (const auto& kv : codecs) {
111 receive_codec_info_[kv.first] = kv.second.clockrate_hz;
112 }
113 }
114 acm_receiver_.SetCodecs(codecs);
115 }
116
ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet)117 void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) {
118 if (!IsPlaying()) {
119 return;
120 }
121
122 RtpPacketReceived rtp_packet_received;
123 rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size());
124
125 // Set payload type's sampling rate before we feed it into ReceiveStatistics.
126 {
127 MutexLock lock(&lock_);
128 const auto& it =
129 receive_codec_info_.find(rtp_packet_received.PayloadType());
130 // If sampling rate info is not available in our received codec set, it
131 // would mean that remote media endpoint is sending incorrect payload id
132 // which can't be processed correctly especially on payload type id in
133 // dynamic range.
134 if (it == receive_codec_info_.end()) {
135 RTC_DLOG(LS_WARNING) << "Unexpected payload id received: "
136 << rtp_packet_received.PayloadType();
137 return;
138 }
139 rtp_packet_received.set_payload_type_frequency(it->second);
140 }
141
142 rtp_receive_statistics_->OnRtpPacket(rtp_packet_received);
143
144 RTPHeader header;
145 rtp_packet_received.GetHeader(&header);
146
147 size_t packet_length = rtp_packet_received.size();
148 if (packet_length < header.headerLength ||
149 (packet_length - header.headerLength) < header.paddingLength) {
150 RTC_DLOG(LS_ERROR) << "Packet length(" << packet_length << ") header("
151 << header.headerLength << ") padding("
152 << header.paddingLength << ")";
153 return;
154 }
155
156 const uint8_t* payload = rtp_packet_received.data() + header.headerLength;
157 size_t payload_length = packet_length - header.headerLength;
158 size_t payload_data_length = payload_length - header.paddingLength;
159 auto data_view = rtc::ArrayView<const uint8_t>(payload, payload_data_length);
160
161 // Push the incoming payload (parsed and ready for decoding) into the ACM.
162 if (acm_receiver_.InsertPacket(header, data_view) != 0) {
163 RTC_DLOG(LS_ERROR) << "AudioIngress::ReceivedRTPPacket() unable to "
164 "push data to the ACM";
165 }
166 }
167
ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet)168 void AudioIngress::ReceivedRTCPPacket(
169 rtc::ArrayView<const uint8_t> rtcp_packet) {
170 // Deliver RTCP packet to RTP/RTCP module for parsing.
171 rtp_rtcp_->IncomingRtcpPacket(rtcp_packet.data(), rtcp_packet.size());
172
173 int64_t rtt = GetRoundTripTime();
174 if (rtt == -1) {
175 // Waiting for valid RTT.
176 return;
177 }
178
179 uint32_t ntp_secs = 0, ntp_frac = 0, rtp_timestamp = 0;
180 if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
181 &rtp_timestamp) != 0) {
182 // Waiting for RTCP.
183 return;
184 }
185
186 {
187 MutexLock lock(&lock_);
188 ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
189 }
190 }
191
GetRoundTripTime()192 int64_t AudioIngress::GetRoundTripTime() {
193 const std::vector<ReportBlockData>& report_data =
194 rtp_rtcp_->GetLatestReportBlockData();
195
196 // If we do not have report block which means remote RTCP hasn't be received
197 // yet, return -1 as to indicate uninitialized value.
198 if (report_data.empty()) {
199 return -1;
200 }
201
202 // We don't know in advance the remote SSRC used by the other end's receiver
203 // reports, so use the SSRC of the first report block as remote SSRC for now.
204 // TODO(natim@webrtc.org): handle the case where remote end is changing ssrc
205 // and update accordingly here.
206 const ReportBlockData& block_data = report_data[0];
207
208 const uint32_t sender_ssrc = block_data.report_block().sender_ssrc;
209
210 if (sender_ssrc != remote_ssrc_.load()) {
211 remote_ssrc_.store(sender_ssrc);
212 rtp_rtcp_->SetRemoteSSRC(sender_ssrc);
213 }
214
215 return (block_data.has_rtt() ? block_data.last_rtt_ms() : -1);
216 }
217
218 } // namespace webrtc
219