1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "media/engine/webrtc_video_engine.h"
12
13 #include <stdio.h>
14
15 #include <algorithm>
16 #include <set>
17 #include <string>
18 #include <utility>
19
20 #include "absl/algorithm/container.h"
21 #include "absl/strings/match.h"
22 #include "api/media_stream_interface.h"
23 #include "api/video/video_codec_constants.h"
24 #include "api/video/video_codec_type.h"
25 #include "api/video_codecs/sdp_video_format.h"
26 #include "api/video_codecs/video_decoder_factory.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "api/video_codecs/video_encoder_factory.h"
29 #include "call/call.h"
30 #include "media/engine/webrtc_media_engine.h"
31 #include "media/engine/webrtc_voice_engine.h"
32 #include "modules/rtp_rtcp/source/rtp_util.h"
33 #include "modules/video_coding/codecs/vp9/svc_config.h"
34 #include "modules/video_coding/svc/scalability_mode_util.h"
35 #include "rtc_base/copy_on_write_buffer.h"
36 #include "rtc_base/experiments/field_trial_parser.h"
37 #include "rtc_base/experiments/field_trial_units.h"
38 #include "rtc_base/logging.h"
39 #include "rtc_base/numerics/safe_conversions.h"
40 #include "rtc_base/strings/string_builder.h"
41 #include "rtc_base/time_utils.h"
42 #include "rtc_base/trace_event.h"
43
44 namespace cricket {
45
46 namespace {
47
48 using ::webrtc::ParseRtpPayloadType;
49 using ::webrtc::ParseRtpSsrc;
50
51 constexpr int64_t kUnsignaledSsrcCooldownMs = rtc::kNumMillisecsPerSec / 2;
52
53 // TODO(bugs.webrtc.org/13166): Remove AV1X when backwards compatibility is not
54 // needed.
55 constexpr char kAv1xCodecName[] = "AV1X";
56
StreamTypeToString(webrtc::VideoSendStream::StreamStats::StreamType type)57 const char* StreamTypeToString(
58 webrtc::VideoSendStream::StreamStats::StreamType type) {
59 switch (type) {
60 case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
61 return "kMedia";
62 case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
63 return "kRtx";
64 case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
65 return "kFlexfec";
66 }
67 return nullptr;
68 }
69
IsEnabled(const webrtc::FieldTrialsView & trials,absl::string_view name)70 bool IsEnabled(const webrtc::FieldTrialsView& trials, absl::string_view name) {
71 return absl::StartsWith(trials.Lookup(name), "Enabled");
72 }
73
IsDisabled(const webrtc::FieldTrialsView & trials,absl::string_view name)74 bool IsDisabled(const webrtc::FieldTrialsView& trials, absl::string_view name) {
75 return absl::StartsWith(trials.Lookup(name), "Disabled");
76 }
77
AddDefaultFeedbackParams(VideoCodec * codec,const webrtc::FieldTrialsView & trials)78 void AddDefaultFeedbackParams(VideoCodec* codec,
79 const webrtc::FieldTrialsView& trials) {
80 // Don't add any feedback params for RED and ULPFEC.
81 if (codec->name == kRedCodecName || codec->name == kUlpfecCodecName)
82 return;
83 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty));
84 codec->AddFeedbackParam(
85 FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
86 // Don't add any more feedback params for FLEXFEC.
87 if (codec->name == kFlexfecCodecName)
88 return;
89 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir));
90 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty));
91 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli));
92 if (codec->name == kVp8CodecName &&
93 IsEnabled(trials, "WebRTC-RtcpLossNotification")) {
94 codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty));
95 }
96 }
97
98 // Helper function to determine whether a codec should use the [35, 63] range.
99 // Should be used when adding new codecs (or variants).
IsCodecValidForLowerRange(const VideoCodec & codec)100 bool IsCodecValidForLowerRange(const VideoCodec& codec) {
101 if (absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName) ||
102 absl::EqualsIgnoreCase(codec.name, kAv1CodecName) ||
103 absl::EqualsIgnoreCase(codec.name, kAv1xCodecName)) {
104 return true;
105 } else if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) {
106 std::string profile_level_id;
107 std::string packetization_mode;
108
109 if (codec.GetParam(kH264FmtpProfileLevelId, &profile_level_id)) {
110 if (absl::StartsWithIgnoreCase(profile_level_id, "4d00")) {
111 if (codec.GetParam(kH264FmtpPacketizationMode, &packetization_mode)) {
112 return packetization_mode == "0";
113 }
114 }
115 // H264 with YUV444.
116 return absl::StartsWithIgnoreCase(profile_level_id, "f400");
117 }
118 } else if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) {
119 std::string profile_id;
120
121 if (codec.GetParam(kVP9ProfileId, &profile_id)) {
122 if (profile_id.compare("1") == 0 || profile_id.compare("3") == 0) {
123 return true;
124 }
125 }
126 }
127 return false;
128 }
129
130 // This function will assign dynamic payload types (in the range [96, 127]
131 // and then [35, 63]) to the input codecs, and also add ULPFEC, RED, FlexFEC,
132 // and associated RTX codecs for recognized codecs (VP8, VP9, H264, and RED).
133 // It will also add default feedback params to the codecs.
134 // is_decoder_factory is needed to keep track of the implict assumption that any
135 // H264 decoder also supports constrained base line profile.
136 // Also, is_decoder_factory is used to decide whether FlexFEC video format
137 // should be advertised as supported.
138 // TODO(kron): Perhaps it is better to move the implicit knowledge to the place
139 // where codecs are negotiated.
140 template <class T>
GetPayloadTypesAndDefaultCodecs(const T * factory,bool is_decoder_factory,bool include_rtx,const webrtc::FieldTrialsView & trials)141 std::vector<VideoCodec> GetPayloadTypesAndDefaultCodecs(
142 const T* factory,
143 bool is_decoder_factory,
144 bool include_rtx,
145 const webrtc::FieldTrialsView& trials) {
146 if (!factory) {
147 return {};
148 }
149
150 std::vector<webrtc::SdpVideoFormat> supported_formats =
151 factory->GetSupportedFormats();
152 if (is_decoder_factory) {
153 AddH264ConstrainedBaselineProfileToSupportedFormats(&supported_formats);
154 }
155
156 if (supported_formats.empty())
157 return std::vector<VideoCodec>();
158
159 supported_formats.push_back(webrtc::SdpVideoFormat(kRedCodecName));
160 supported_formats.push_back(webrtc::SdpVideoFormat(kUlpfecCodecName));
161
162 // flexfec-03 is supported as
163 // - receive codec unless WebRTC-FlexFEC-03-Advertised is disabled
164 // - send codec if WebRTC-FlexFEC-03-Advertised is enabled
165 if ((is_decoder_factory &&
166 !IsDisabled(trials, "WebRTC-FlexFEC-03-Advertised")) ||
167 (!is_decoder_factory &&
168 IsEnabled(trials, "WebRTC-FlexFEC-03-Advertised"))) {
169 webrtc::SdpVideoFormat flexfec_format(kFlexfecCodecName);
170 // This value is currently arbitrarily set to 10 seconds. (The unit
171 // is microseconds.) This parameter MUST be present in the SDP, but
172 // we never use the actual value anywhere in our code however.
173 // TODO(brandtr): Consider honouring this value in the sender and receiver.
174 flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}};
175 supported_formats.push_back(flexfec_format);
176 }
177
178 // Due to interoperability issues with old Chrome/WebRTC versions that
179 // ignore the [35, 63] range prefer the lower range for new codecs.
180 static const int kFirstDynamicPayloadTypeLowerRange = 35;
181 static const int kLastDynamicPayloadTypeLowerRange = 63;
182
183 static const int kFirstDynamicPayloadTypeUpperRange = 96;
184 static const int kLastDynamicPayloadTypeUpperRange = 127;
185 int payload_type_upper = kFirstDynamicPayloadTypeUpperRange;
186 int payload_type_lower = kFirstDynamicPayloadTypeLowerRange;
187
188 std::vector<VideoCodec> output_codecs;
189 for (const webrtc::SdpVideoFormat& format : supported_formats) {
190 VideoCodec codec(format);
191 bool isFecCodec = absl::EqualsIgnoreCase(codec.name, kUlpfecCodecName) ||
192 absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName);
193
194 // Check if we ran out of payload types.
195 if (payload_type_lower > kLastDynamicPayloadTypeLowerRange) {
196 // TODO(https://bugs.chromium.org/p/webrtc/issues/detail?id=12248):
197 // return an error.
198 RTC_LOG(LS_ERROR) << "Out of dynamic payload types [35,63] after "
199 "fallback from [96, 127], skipping the rest.";
200 RTC_DCHECK_EQ(payload_type_upper, kLastDynamicPayloadTypeUpperRange);
201 break;
202 }
203
204 // Lower range gets used for "new" codecs or when running out of payload
205 // types in the upper range.
206 if (IsCodecValidForLowerRange(codec) ||
207 payload_type_upper >= kLastDynamicPayloadTypeUpperRange) {
208 codec.id = payload_type_lower++;
209 } else {
210 codec.id = payload_type_upper++;
211 }
212 AddDefaultFeedbackParams(&codec, trials);
213 output_codecs.push_back(codec);
214
215 // Add associated RTX codec for non-FEC codecs.
216 if (include_rtx) {
217 if (!isFecCodec) {
218 // Check if we ran out of payload types.
219 if (payload_type_lower > kLastDynamicPayloadTypeLowerRange) {
220 // TODO(https://bugs.chromium.org/p/webrtc/issues/detail?id=12248):
221 // return an error.
222 RTC_LOG(LS_ERROR) << "Out of dynamic payload types [35,63] after "
223 "fallback from [96, 127], skipping the rest.";
224 RTC_DCHECK_EQ(payload_type_upper, kLastDynamicPayloadTypeUpperRange);
225 break;
226 }
227 if (IsCodecValidForLowerRange(codec) ||
228 payload_type_upper >= kLastDynamicPayloadTypeUpperRange) {
229 output_codecs.push_back(
230 VideoCodec::CreateRtxCodec(payload_type_lower++, codec.id));
231 } else {
232 output_codecs.push_back(
233 VideoCodec::CreateRtxCodec(payload_type_upper++, codec.id));
234 }
235 }
236 }
237 }
238 return output_codecs;
239 }
240
CodecVectorToString(const std::vector<VideoCodec> & codecs)241 static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) {
242 rtc::StringBuilder out;
243 out << "{";
244 for (size_t i = 0; i < codecs.size(); ++i) {
245 out << codecs[i].ToString();
246 if (i != codecs.size() - 1) {
247 out << ", ";
248 }
249 }
250 out << "}";
251 return out.Release();
252 }
253
ValidateCodecFormats(const std::vector<VideoCodec> & codecs)254 static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) {
255 bool has_video = false;
256 for (size_t i = 0; i < codecs.size(); ++i) {
257 if (!codecs[i].ValidateCodecFormat()) {
258 return false;
259 }
260 if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) {
261 has_video = true;
262 }
263 }
264 if (!has_video) {
265 RTC_LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: "
266 << CodecVectorToString(codecs);
267 return false;
268 }
269 return true;
270 }
271
ValidateStreamParams(const StreamParams & sp)272 static bool ValidateStreamParams(const StreamParams& sp) {
273 if (sp.ssrcs.empty()) {
274 RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString();
275 return false;
276 }
277
278 std::vector<uint32_t> primary_ssrcs;
279 sp.GetPrimarySsrcs(&primary_ssrcs);
280 std::vector<uint32_t> rtx_ssrcs;
281 sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs);
282 for (uint32_t rtx_ssrc : rtx_ssrcs) {
283 bool rtx_ssrc_present = false;
284 for (uint32_t sp_ssrc : sp.ssrcs) {
285 if (sp_ssrc == rtx_ssrc) {
286 rtx_ssrc_present = true;
287 break;
288 }
289 }
290 if (!rtx_ssrc_present) {
291 RTC_LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc
292 << "' missing from StreamParams ssrcs: "
293 << sp.ToString();
294 return false;
295 }
296 }
297 if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) {
298 RTC_LOG(LS_ERROR)
299 << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): "
300 << sp.ToString();
301 return false;
302 }
303
304 return true;
305 }
306
307 // Returns true if the given codec is disallowed from doing simulcast.
IsCodecDisabledForSimulcast(const std::string & codec_name,const webrtc::FieldTrialsView & trials)308 bool IsCodecDisabledForSimulcast(const std::string& codec_name,
309 const webrtc::FieldTrialsView& trials) {
310 if (absl::EqualsIgnoreCase(codec_name, kVp9CodecName) ||
311 absl::EqualsIgnoreCase(codec_name, kAv1CodecName)) {
312 return true;
313 }
314
315 if (absl::EqualsIgnoreCase(codec_name, kH264CodecName)) {
316 return absl::StartsWith(trials.Lookup("WebRTC-H264Simulcast"), "Disabled");
317 }
318
319 return false;
320 }
321
322 // Returns its smallest positive argument. If neither argument is positive,
323 // returns an arbitrary nonpositive value.
MinPositive(int a,int b)324 int MinPositive(int a, int b) {
325 if (a <= 0) {
326 return b;
327 }
328 if (b <= 0) {
329 return a;
330 }
331 return std::min(a, b);
332 }
333
IsLayerActive(const webrtc::RtpEncodingParameters & layer)334 bool IsLayerActive(const webrtc::RtpEncodingParameters& layer) {
335 return layer.active &&
336 (!layer.max_bitrate_bps || *layer.max_bitrate_bps > 0) &&
337 (!layer.max_framerate || *layer.max_framerate > 0);
338 }
339
NumActiveStreams(const webrtc::RtpParameters & rtp_parameters)340 int NumActiveStreams(const webrtc::RtpParameters& rtp_parameters) {
341 int res = 0;
342 for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) {
343 if (rtp_parameters.encodings[i].active) {
344 ++res;
345 }
346 }
347 return res;
348 }
349
NumSpatialLayersFromEncoding(const webrtc::RtpParameters & rtp_parameters,size_t idx)350 absl::optional<int> NumSpatialLayersFromEncoding(
351 const webrtc::RtpParameters& rtp_parameters,
352 size_t idx) {
353 if (idx >= rtp_parameters.encodings.size())
354 return absl::nullopt;
355
356 absl::optional<webrtc::ScalabilityMode> scalability_mode =
357 webrtc::ScalabilityModeFromString(
358 rtp_parameters.encodings[idx].scalability_mode.value_or(""));
359 return scalability_mode
360 ? absl::optional<int>(
361 ScalabilityModeToNumSpatialLayers(*scalability_mode))
362 : absl::nullopt;
363 }
364
365 std::map<uint32_t, webrtc::VideoSendStream::StreamStats>
MergeInfoAboutOutboundRtpSubstreams(const std::map<uint32_t,webrtc::VideoSendStream::StreamStats> & substreams)366 MergeInfoAboutOutboundRtpSubstreams(
367 const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>&
368 substreams) {
369 std::map<uint32_t, webrtc::VideoSendStream::StreamStats> rtp_substreams;
370 // Add substreams for all RTP media streams.
371 for (const auto& pair : substreams) {
372 uint32_t ssrc = pair.first;
373 const webrtc::VideoSendStream::StreamStats& substream = pair.second;
374 switch (substream.type) {
375 case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
376 break;
377 case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
378 case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
379 continue;
380 }
381 rtp_substreams.insert(std::make_pair(ssrc, substream));
382 }
383 // Complement the kMedia substream stats with the associated kRtx and kFlexfec
384 // substream stats.
385 for (const auto& pair : substreams) {
386 switch (pair.second.type) {
387 case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
388 continue;
389 case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
390 case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
391 break;
392 }
393 // The associated substream is an RTX or FlexFEC substream that is
394 // referencing an RTP media substream.
395 const webrtc::VideoSendStream::StreamStats& associated_substream =
396 pair.second;
397 RTC_DCHECK(associated_substream.referenced_media_ssrc.has_value());
398 uint32_t media_ssrc = associated_substream.referenced_media_ssrc.value();
399 if (substreams.find(media_ssrc) == substreams.end()) {
400 RTC_LOG(LS_WARNING) << "Substream [ssrc: " << pair.first << ", type: "
401 << StreamTypeToString(associated_substream.type)
402 << "] is associated with a media ssrc (" << media_ssrc
403 << ") that does not have StreamStats. Ignoring its "
404 << "RTP stats.";
405 continue;
406 }
407 webrtc::VideoSendStream::StreamStats& rtp_substream =
408 rtp_substreams[media_ssrc];
409
410 // We only merge `rtp_stats`. All other metrics are not applicable for RTX
411 // and FlexFEC.
412 // TODO(hbos): kRtx and kFlexfec stats should use a separate struct to make
413 // it clear what is or is not applicable.
414 rtp_substream.rtp_stats.Add(associated_substream.rtp_stats);
415 }
416 return rtp_substreams;
417 }
418
IsActiveFromEncodings(absl::optional<uint32_t> ssrc,const std::vector<webrtc::RtpEncodingParameters> & encodings)419 bool IsActiveFromEncodings(
420 absl::optional<uint32_t> ssrc,
421 const std::vector<webrtc::RtpEncodingParameters>& encodings) {
422 if (ssrc.has_value()) {
423 // Report the `active` value of a specific ssrc, or false if an encoding
424 // with this ssrc does not exist.
425 auto encoding_it = std::find_if(
426 encodings.begin(), encodings.end(),
427 [ssrc = ssrc.value()](const webrtc::RtpEncodingParameters& encoding) {
428 return encoding.ssrc.has_value() && encoding.ssrc.value() == ssrc;
429 });
430 return encoding_it != encodings.end() ? encoding_it->active : false;
431 }
432 // If `ssrc` is not specified then any encoding being active counts as active.
433 for (const auto& encoding : encodings) {
434 if (encoding.active) {
435 return true;
436 }
437 }
438 return false;
439 }
440
441 } // namespace
442
443 // This constant is really an on/off, lower-level configurable NACK history
444 // duration hasn't been implemented.
445 static const int kNackHistoryMs = 1000;
446
447 static const int kDefaultRtcpReceiverReportSsrc = 1;
448
449 // Minimum time interval for logging stats.
450 static const int64_t kStatsLogIntervalMs = 10000;
451
452 std::map<uint32_t, webrtc::VideoSendStream::StreamStats>
MergeInfoAboutOutboundRtpSubstreamsForTesting(const std::map<uint32_t,webrtc::VideoSendStream::StreamStats> & substreams)453 MergeInfoAboutOutboundRtpSubstreamsForTesting(
454 const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>&
455 substreams) {
456 return MergeInfoAboutOutboundRtpSubstreams(substreams);
457 }
458
459 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
ConfigureVideoEncoderSettings(const VideoCodec & codec)460 WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings(
461 const VideoCodec& codec) {
462 RTC_DCHECK_RUN_ON(&thread_checker_);
463 bool is_screencast = parameters_.options.is_screencast.value_or(false);
464 // No automatic resizing when using simulcast or screencast, or when
465 // disabled by field trial flag.
466 bool automatic_resize = !disable_automatic_resize_ && !is_screencast &&
467 (parameters_.config.rtp.ssrcs.size() == 1 ||
468 NumActiveStreams(rtp_parameters_) == 1);
469
470 bool denoising;
471 bool codec_default_denoising = false;
472 if (is_screencast) {
473 denoising = false;
474 } else {
475 // Use codec default if video_noise_reduction is unset.
476 codec_default_denoising = !parameters_.options.video_noise_reduction;
477 denoising = parameters_.options.video_noise_reduction.value_or(false);
478 }
479
480 if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) {
481 return nullptr;
482 }
483 if (absl::EqualsIgnoreCase(codec.name, kVp8CodecName)) {
484 webrtc::VideoCodecVP8 vp8_settings =
485 webrtc::VideoEncoder::GetDefaultVp8Settings();
486 vp8_settings.automaticResizeOn = automatic_resize;
487 // VP8 denoising is enabled by default.
488 vp8_settings.denoisingOn = codec_default_denoising ? true : denoising;
489 return rtc::make_ref_counted<
490 webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
491 }
492 if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) {
493 webrtc::VideoCodecVP9 vp9_settings =
494 webrtc::VideoEncoder::GetDefaultVp9Settings();
495
496 vp9_settings.numberOfSpatialLayers = std::min<unsigned char>(
497 parameters_.config.rtp.ssrcs.size(), kConferenceMaxNumSpatialLayers);
498 vp9_settings.numberOfTemporalLayers =
499 std::min<unsigned char>(parameters_.config.rtp.ssrcs.size() > 1
500 ? kConferenceDefaultNumTemporalLayers
501 : 1,
502 kConferenceMaxNumTemporalLayers);
503
504 // VP9 denoising is disabled by default.
505 vp9_settings.denoisingOn = codec_default_denoising ? true : denoising;
506 // Disable automatic resize if more than one spatial layer is requested.
507 bool vp9_automatic_resize = automatic_resize;
508 absl::optional<int> num_spatial_layers =
509 NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0);
510 if (num_spatial_layers && *num_spatial_layers > 1) {
511 vp9_automatic_resize = false;
512 }
513 vp9_settings.automaticResizeOn = vp9_automatic_resize;
514 if (!is_screencast) {
515 webrtc::FieldTrialFlag interlayer_pred_experiment_enabled("Enabled");
516 webrtc::FieldTrialEnum<webrtc::InterLayerPredMode> inter_layer_pred_mode(
517 "inter_layer_pred_mode", webrtc::InterLayerPredMode::kOnKeyPic,
518 {{"off", webrtc::InterLayerPredMode::kOff},
519 {"on", webrtc::InterLayerPredMode::kOn},
520 {"onkeypic", webrtc::InterLayerPredMode::kOnKeyPic}});
521 webrtc::FieldTrialFlag force_flexible_mode("FlexibleMode");
522 webrtc::ParseFieldTrial(
523 {&interlayer_pred_experiment_enabled, &inter_layer_pred_mode,
524 &force_flexible_mode},
525 call_->trials().Lookup("WebRTC-Vp9InterLayerPred"));
526 if (interlayer_pred_experiment_enabled) {
527 vp9_settings.interLayerPred = inter_layer_pred_mode;
528 } else {
529 // Limit inter-layer prediction to key pictures by default.
530 vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic;
531 }
532 vp9_settings.flexibleMode = force_flexible_mode.Get();
533 } else {
534 // Multiple spatial layers vp9 screenshare needs flexible mode.
535 vp9_settings.flexibleMode = vp9_settings.numberOfSpatialLayers > 1;
536 vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOn;
537 }
538 return rtc::make_ref_counted<
539 webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
540 }
541 return nullptr;
542 }
543
DefaultUnsignalledSsrcHandler()544 DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler()
545 : default_sink_(nullptr) {}
546
OnUnsignalledSsrc(WebRtcVideoChannel * channel,uint32_t ssrc,absl::optional<uint32_t> rtx_ssrc)547 UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc(
548 WebRtcVideoChannel* channel,
549 uint32_t ssrc,
550 absl::optional<uint32_t> rtx_ssrc) {
551 absl::optional<uint32_t> default_recv_ssrc =
552 channel->GetDefaultReceiveStreamSsrc();
553
554 if (default_recv_ssrc) {
555 RTC_LOG(LS_INFO) << "Destroying old default receive stream for SSRC="
556 << ssrc << ".";
557 channel->RemoveRecvStream(*default_recv_ssrc);
558 }
559
560 StreamParams sp = channel->unsignaled_stream_params();
561 sp.ssrcs.push_back(ssrc);
562 if (rtx_ssrc) {
563 sp.AddFidSsrc(ssrc, *rtx_ssrc);
564 }
565 RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc
566 << ".";
567 if (!channel->AddRecvStream(sp, /*default_stream=*/true)) {
568 RTC_LOG(LS_WARNING) << "Could not create default receive stream.";
569 }
570
571 // SSRC 0 returns default_recv_base_minimum_delay_ms.
572 const int unsignaled_ssrc = 0;
573 int default_recv_base_minimum_delay_ms =
574 channel->GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0);
575 // Set base minimum delay if it was set before for the default receive stream.
576 channel->SetBaseMinimumPlayoutDelayMs(ssrc,
577 default_recv_base_minimum_delay_ms);
578 channel->SetSink(ssrc, default_sink_);
579 return kDeliverPacket;
580 }
581
582 rtc::VideoSinkInterface<webrtc::VideoFrame>*
GetDefaultSink() const583 DefaultUnsignalledSsrcHandler::GetDefaultSink() const {
584 return default_sink_;
585 }
586
SetDefaultSink(WebRtcVideoChannel * channel,rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)587 void DefaultUnsignalledSsrcHandler::SetDefaultSink(
588 WebRtcVideoChannel* channel,
589 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
590 default_sink_ = sink;
591 absl::optional<uint32_t> default_recv_ssrc =
592 channel->GetDefaultReceiveStreamSsrc();
593 if (default_recv_ssrc) {
594 channel->SetSink(*default_recv_ssrc, default_sink_);
595 }
596 }
597
WebRtcVideoEngine(std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,const webrtc::FieldTrialsView & trials)598 WebRtcVideoEngine::WebRtcVideoEngine(
599 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
600 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
601 const webrtc::FieldTrialsView& trials)
602 : decoder_factory_(std::move(video_decoder_factory)),
603 encoder_factory_(std::move(video_encoder_factory)),
604 trials_(trials) {
605 RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()";
606 }
607
~WebRtcVideoEngine()608 WebRtcVideoEngine::~WebRtcVideoEngine() {
609 RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
610 }
611
CreateMediaChannel(webrtc::Call * call,const MediaConfig & config,const VideoOptions & options,const webrtc::CryptoOptions & crypto_options,webrtc::VideoBitrateAllocatorFactory * video_bitrate_allocator_factory)612 VideoMediaChannel* WebRtcVideoEngine::CreateMediaChannel(
613 webrtc::Call* call,
614 const MediaConfig& config,
615 const VideoOptions& options,
616 const webrtc::CryptoOptions& crypto_options,
617 webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) {
618 RTC_LOG(LS_INFO) << "CreateMediaChannel. Options: " << options.ToString();
619 return new WebRtcVideoChannel(call, config, options, crypto_options,
620 encoder_factory_.get(), decoder_factory_.get(),
621 video_bitrate_allocator_factory);
622 }
send_codecs(bool include_rtx) const623 std::vector<VideoCodec> WebRtcVideoEngine::send_codecs(bool include_rtx) const {
624 return GetPayloadTypesAndDefaultCodecs(encoder_factory_.get(),
625 /*is_decoder_factory=*/false,
626 include_rtx, trials_);
627 }
628
recv_codecs(bool include_rtx) const629 std::vector<VideoCodec> WebRtcVideoEngine::recv_codecs(bool include_rtx) const {
630 return GetPayloadTypesAndDefaultCodecs(decoder_factory_.get(),
631 /*is_decoder_factory=*/true,
632 include_rtx, trials_);
633 }
634
635 std::vector<webrtc::RtpHeaderExtensionCapability>
GetRtpHeaderExtensions() const636 WebRtcVideoEngine::GetRtpHeaderExtensions() const {
637 std::vector<webrtc::RtpHeaderExtensionCapability> result;
638 int id = 1;
639 for (const auto& uri :
640 {webrtc::RtpExtension::kTimestampOffsetUri,
641 webrtc::RtpExtension::kAbsSendTimeUri,
642 webrtc::RtpExtension::kVideoRotationUri,
643 webrtc::RtpExtension::kTransportSequenceNumberUri,
644 webrtc::RtpExtension::kPlayoutDelayUri,
645 webrtc::RtpExtension::kVideoContentTypeUri,
646 webrtc::RtpExtension::kVideoTimingUri,
647 webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri,
648 webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) {
649 result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
650 }
651 result.emplace_back(webrtc::RtpExtension::kGenericFrameDescriptorUri00, id++,
652 IsEnabled(trials_, "WebRTC-GenericDescriptorAdvertised")
653 ? webrtc::RtpTransceiverDirection::kSendRecv
654 : webrtc::RtpTransceiverDirection::kStopped);
655 result.emplace_back(
656 webrtc::RtpExtension::kDependencyDescriptorUri, id++,
657 IsEnabled(trials_, "WebRTC-DependencyDescriptorAdvertised")
658 ? webrtc::RtpTransceiverDirection::kSendRecv
659 : webrtc::RtpTransceiverDirection::kStopped);
660
661 result.emplace_back(
662 webrtc::RtpExtension::kVideoLayersAllocationUri, id++,
663 IsEnabled(trials_, "WebRTC-VideoLayersAllocationAdvertised")
664 ? webrtc::RtpTransceiverDirection::kSendRecv
665 : webrtc::RtpTransceiverDirection::kStopped);
666
667 result.emplace_back(
668 webrtc::RtpExtension::kVideoFrameTrackingIdUri, id++,
669 IsEnabled(trials_, "WebRTC-VideoFrameTrackingIdAdvertised")
670 ? webrtc::RtpTransceiverDirection::kSendRecv
671 : webrtc::RtpTransceiverDirection::kStopped);
672
673 return result;
674 }
675
WebRtcVideoChannel(webrtc::Call * call,const MediaConfig & config,const VideoOptions & options,const webrtc::CryptoOptions & crypto_options,webrtc::VideoEncoderFactory * encoder_factory,webrtc::VideoDecoderFactory * decoder_factory,webrtc::VideoBitrateAllocatorFactory * bitrate_allocator_factory)676 WebRtcVideoChannel::WebRtcVideoChannel(
677 webrtc::Call* call,
678 const MediaConfig& config,
679 const VideoOptions& options,
680 const webrtc::CryptoOptions& crypto_options,
681 webrtc::VideoEncoderFactory* encoder_factory,
682 webrtc::VideoDecoderFactory* decoder_factory,
683 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
684 : VideoMediaChannel(call->network_thread(), config.enable_dscp),
685 worker_thread_(call->worker_thread()),
686 call_(call),
687 unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
688 video_config_(config.video),
689 encoder_factory_(encoder_factory),
690 decoder_factory_(decoder_factory),
691 bitrate_allocator_factory_(bitrate_allocator_factory),
692 default_send_options_(options),
693 last_stats_log_ms_(-1),
694 discard_unknown_ssrc_packets_(
695 IsEnabled(call_->trials(),
696 "WebRTC-Video-DiscardPacketsWithUnknownSsrc")),
697 crypto_options_(crypto_options),
698 unknown_ssrc_packet_buffer_(
699 IsEnabled(call_->trials(),
700 "WebRTC-Video-BufferPacketsWithUnknownSsrc")
701 ? new UnhandledPacketsBuffer()
702 : nullptr) {
703 RTC_DCHECK_RUN_ON(&thread_checker_);
704 network_thread_checker_.Detach();
705
706 rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc;
707 sending_ = false;
708 recv_codecs_ = MapCodecs(GetPayloadTypesAndDefaultCodecs(
709 decoder_factory_, /*is_decoder_factory=*/true,
710 /*include_rtx=*/true, call_->trials()));
711 recv_flexfec_payload_type_ =
712 recv_codecs_.empty() ? 0 : recv_codecs_.front().flexfec_payload_type;
713 }
714
~WebRtcVideoChannel()715 WebRtcVideoChannel::~WebRtcVideoChannel() {
716 for (auto& kv : send_streams_)
717 delete kv.second;
718 for (auto& kv : receive_streams_)
719 delete kv.second;
720 }
721
722 std::vector<WebRtcVideoChannel::VideoCodecSettings>
SelectSendVideoCodecs(const std::vector<VideoCodecSettings> & remote_mapped_codecs) const723 WebRtcVideoChannel::SelectSendVideoCodecs(
724 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
725 std::vector<webrtc::SdpVideoFormat> sdp_formats =
726 encoder_factory_ ? encoder_factory_->GetImplementations()
727 : std::vector<webrtc::SdpVideoFormat>();
728
729 // The returned vector holds the VideoCodecSettings in term of preference.
730 // They are orderd by receive codec preference first and local implementation
731 // preference second.
732 std::vector<VideoCodecSettings> encoders;
733 for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) {
734 for (auto format_it = sdp_formats.begin();
735 format_it != sdp_formats.end();) {
736 // For H264, we will limit the encode level to the remote offered level
737 // regardless if level asymmetry is allowed or not. This is strictly not
738 // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
739 // since we should limit the encode level to the lower of local and remote
740 // level when level asymmetry is not allowed.
741 if (format_it->IsSameCodec(
742 {remote_codec.codec.name, remote_codec.codec.params})) {
743 encoders.push_back(remote_codec);
744
745 // To allow the VideoEncoderFactory to keep information about which
746 // implementation to instantitate when CreateEncoder is called the two
747 // parmeter sets are merged.
748 encoders.back().codec.params.insert(format_it->parameters.begin(),
749 format_it->parameters.end());
750
751 format_it = sdp_formats.erase(format_it);
752 } else {
753 ++format_it;
754 }
755 }
756 }
757
758 return encoders;
759 }
760
NonFlexfecReceiveCodecsHaveChanged(std::vector<VideoCodecSettings> before,std::vector<VideoCodecSettings> after)761 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
762 std::vector<VideoCodecSettings> before,
763 std::vector<VideoCodecSettings> after) {
764 // The receive codec order doesn't matter, so we sort the codecs before
765 // comparing. This is necessary because currently the
766 // only way to change the send codec is to munge SDP, which causes
767 // the receive codec list to change order, which causes the streams
768 // to be recreates which causes a "blink" of black video. In order
769 // to support munging the SDP in this way without recreating receive
770 // streams, we ignore the order of the received codecs so that
771 // changing the order doesn't cause this "blink".
772 auto comparison = [](const VideoCodecSettings& codec1,
773 const VideoCodecSettings& codec2) {
774 return codec1.codec.id > codec2.codec.id;
775 };
776 absl::c_sort(before, comparison);
777 absl::c_sort(after, comparison);
778
779 // Changes in FlexFEC payload type are handled separately in
780 // WebRtcVideoChannel::GetChangedRecvParameters, so disregard FlexFEC in the
781 // comparison here.
782 return !absl::c_equal(before, after,
783 VideoCodecSettings::EqualsDisregardingFlexfec);
784 }
785
GetChangedSendParameters(const VideoSendParameters & params,ChangedSendParameters * changed_params) const786 bool WebRtcVideoChannel::GetChangedSendParameters(
787 const VideoSendParameters& params,
788 ChangedSendParameters* changed_params) const {
789 if (!ValidateCodecFormats(params.codecs) ||
790 !ValidateRtpExtensions(params.extensions, send_rtp_extensions_)) {
791 return false;
792 }
793
794 std::vector<VideoCodecSettings> negotiated_codecs =
795 SelectSendVideoCodecs(MapCodecs(params.codecs));
796
797 // We should only fail here if send direction is enabled.
798 if (params.is_stream_active && negotiated_codecs.empty()) {
799 RTC_LOG(LS_ERROR) << "No video codecs supported.";
800 return false;
801 }
802
803 // Never enable sending FlexFEC, unless we are in the experiment.
804 if (!IsEnabled(call_->trials(), "WebRTC-FlexFEC-03")) {
805 for (VideoCodecSettings& codec : negotiated_codecs)
806 codec.flexfec_payload_type = -1;
807 }
808
809 if (negotiated_codecs_ != negotiated_codecs) {
810 if (negotiated_codecs.empty()) {
811 changed_params->send_codec = absl::nullopt;
812 } else if (send_codec_ != negotiated_codecs.front()) {
813 changed_params->send_codec = negotiated_codecs.front();
814 }
815 changed_params->negotiated_codecs = std::move(negotiated_codecs);
816 }
817
818 // Handle RTP header extensions.
819 if (params.extmap_allow_mixed != ExtmapAllowMixed()) {
820 changed_params->extmap_allow_mixed = params.extmap_allow_mixed;
821 }
822 std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
823 params.extensions, webrtc::RtpExtension::IsSupportedForVideo, true,
824 call_->trials());
825 if (send_rtp_extensions_ != filtered_extensions) {
826 changed_params->rtp_header_extensions =
827 absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
828 }
829
830 if (params.mid != send_params_.mid) {
831 changed_params->mid = params.mid;
832 }
833
834 // Handle max bitrate.
835 if (params.max_bandwidth_bps != send_params_.max_bandwidth_bps &&
836 params.max_bandwidth_bps >= -1) {
837 // 0 or -1 uncaps max bitrate.
838 // TODO(pbos): Reconsider how 0 should be treated. It is not mentioned as a
839 // special value and might very well be used for stopping sending.
840 changed_params->max_bandwidth_bps =
841 params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps;
842 }
843
844 // Handle conference mode.
845 if (params.conference_mode != send_params_.conference_mode) {
846 changed_params->conference_mode = params.conference_mode;
847 }
848
849 // Handle RTCP mode.
850 if (params.rtcp.reduced_size != send_params_.rtcp.reduced_size) {
851 changed_params->rtcp_mode = params.rtcp.reduced_size
852 ? webrtc::RtcpMode::kReducedSize
853 : webrtc::RtcpMode::kCompound;
854 }
855
856 return true;
857 }
858
SetSendParameters(const VideoSendParameters & params)859 bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
860 RTC_DCHECK_RUN_ON(&thread_checker_);
861 TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSendParameters");
862 RTC_LOG(LS_INFO) << "SetSendParameters: " << params.ToString();
863 ChangedSendParameters changed_params;
864 if (!GetChangedSendParameters(params, &changed_params)) {
865 return false;
866 }
867
868 if (changed_params.negotiated_codecs) {
869 for (const auto& send_codec : *changed_params.negotiated_codecs)
870 RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString();
871 }
872
873 send_params_ = params;
874 return ApplyChangedParams(changed_params);
875 }
876
RequestEncoderFallback()877 void WebRtcVideoChannel::RequestEncoderFallback() {
878 if (!worker_thread_->IsCurrent()) {
879 worker_thread_->PostTask(
880 SafeTask(task_safety_.flag(), [this] { RequestEncoderFallback(); }));
881 return;
882 }
883
884 RTC_DCHECK_RUN_ON(&thread_checker_);
885 if (negotiated_codecs_.size() <= 1) {
886 RTC_LOG(LS_WARNING) << "Encoder failed but no fallback codec is available";
887 return;
888 }
889
890 ChangedSendParameters params;
891 params.negotiated_codecs = negotiated_codecs_;
892 params.negotiated_codecs->erase(params.negotiated_codecs->begin());
893 params.send_codec = params.negotiated_codecs->front();
894 ApplyChangedParams(params);
895 }
896
RequestEncoderSwitch(const webrtc::SdpVideoFormat & format,bool allow_default_fallback)897 void WebRtcVideoChannel::RequestEncoderSwitch(
898 const webrtc::SdpVideoFormat& format,
899 bool allow_default_fallback) {
900 if (!worker_thread_->IsCurrent()) {
901 worker_thread_->PostTask(
902 SafeTask(task_safety_.flag(), [this, format, allow_default_fallback] {
903 RequestEncoderSwitch(format, allow_default_fallback);
904 }));
905 return;
906 }
907
908 RTC_DCHECK_RUN_ON(&thread_checker_);
909
910 for (const VideoCodecSettings& codec_setting : negotiated_codecs_) {
911 if (format.IsSameCodec(
912 {codec_setting.codec.name, codec_setting.codec.params})) {
913 VideoCodecSettings new_codec_setting = codec_setting;
914 for (const auto& kv : format.parameters) {
915 new_codec_setting.codec.params[kv.first] = kv.second;
916 }
917
918 if (send_codec_ == new_codec_setting) {
919 // Already using this codec, no switch required.
920 return;
921 }
922
923 ChangedSendParameters params;
924 params.send_codec = new_codec_setting;
925 ApplyChangedParams(params);
926 return;
927 }
928 }
929
930 RTC_LOG(LS_WARNING) << "Failed to switch encoder to: " << format.ToString()
931 << ". Is default fallback allowed: "
932 << allow_default_fallback;
933
934 if (allow_default_fallback) {
935 RequestEncoderFallback();
936 }
937 }
938
ApplyChangedParams(const ChangedSendParameters & changed_params)939 bool WebRtcVideoChannel::ApplyChangedParams(
940 const ChangedSendParameters& changed_params) {
941 RTC_DCHECK_RUN_ON(&thread_checker_);
942 if (changed_params.negotiated_codecs)
943 negotiated_codecs_ = *changed_params.negotiated_codecs;
944
945 if (changed_params.send_codec)
946 send_codec_ = changed_params.send_codec;
947
948 if (changed_params.extmap_allow_mixed) {
949 SetExtmapAllowMixed(*changed_params.extmap_allow_mixed);
950 }
951 if (changed_params.rtp_header_extensions) {
952 send_rtp_extensions_ = *changed_params.rtp_header_extensions;
953 }
954
955 if (changed_params.send_codec || changed_params.max_bandwidth_bps) {
956 if (send_params_.max_bandwidth_bps == -1) {
957 // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is
958 // -1, which corresponds to no "b=AS" attribute in SDP. Note that the
959 // global max bitrate may be set below in GetBitrateConfigForCodec, from
960 // the codec max bitrate.
961 // TODO(pbos): This should be reconsidered (codec max bitrate should
962 // probably not affect global call max bitrate).
963 bitrate_config_.max_bitrate_bps = -1;
964 }
965
966 if (send_codec_) {
967 // TODO(holmer): Changing the codec parameters shouldn't necessarily mean
968 // that we change the min/max of bandwidth estimation. Reevaluate this.
969 bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec);
970 if (!changed_params.send_codec) {
971 // If the codec isn't changing, set the start bitrate to -1 which means
972 // "unchanged" so that BWE isn't affected.
973 bitrate_config_.start_bitrate_bps = -1;
974 }
975 }
976
977 if (send_params_.max_bandwidth_bps >= 0) {
978 // Note that max_bandwidth_bps intentionally takes priority over the
979 // bitrate config for the codec. This allows FEC to be applied above the
980 // codec target bitrate.
981 // TODO(pbos): Figure out whether b=AS means max bitrate for this
982 // WebRtcVideoChannel (in which case we're good), or per sender (SSRC),
983 // in which case this should not set a BitrateConstraints but rather
984 // reconfigure all senders.
985 bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0
986 ? -1
987 : send_params_.max_bandwidth_bps;
988 }
989
990 call_->GetTransportControllerSend()->SetSdpBitrateParameters(
991 bitrate_config_);
992 }
993
994 for (auto& kv : send_streams_) {
995 kv.second->SetSendParameters(changed_params);
996 }
997 if (changed_params.send_codec || changed_params.rtcp_mode) {
998 // Update receive feedback parameters from new codec or RTCP mode.
999 RTC_LOG(LS_INFO)
1000 << "SetFeedbackParameters on all the receive streams because the send "
1001 "codec or RTCP mode has changed.";
1002 for (auto& kv : receive_streams_) {
1003 RTC_DCHECK(kv.second != nullptr);
1004 kv.second->SetFeedbackParameters(
1005 HasLntf(send_codec_->codec), HasNack(send_codec_->codec),
1006 HasTransportCc(send_codec_->codec),
1007 send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
1008 : webrtc::RtcpMode::kCompound,
1009 send_codec_->rtx_time);
1010 }
1011 }
1012 return true;
1013 }
1014
GetRtpSendParameters(uint32_t ssrc) const1015 webrtc::RtpParameters WebRtcVideoChannel::GetRtpSendParameters(
1016 uint32_t ssrc) const {
1017 RTC_DCHECK_RUN_ON(&thread_checker_);
1018 auto it = send_streams_.find(ssrc);
1019 if (it == send_streams_.end()) {
1020 RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream "
1021 "with ssrc "
1022 << ssrc << " which doesn't exist.";
1023 return webrtc::RtpParameters();
1024 }
1025
1026 webrtc::RtpParameters rtp_params = it->second->GetRtpParameters();
1027 // Need to add the common list of codecs to the send stream-specific
1028 // RTP parameters.
1029 for (const VideoCodec& codec : send_params_.codecs) {
1030 if (send_codec_ && send_codec_->codec.id == codec.id) {
1031 // Put the current send codec to the front of the codecs list.
1032 RTC_DCHECK_EQ(codec.name, send_codec_->codec.name);
1033 rtp_params.codecs.insert(rtp_params.codecs.begin(),
1034 codec.ToCodecParameters());
1035 } else {
1036 rtp_params.codecs.push_back(codec.ToCodecParameters());
1037 }
1038 }
1039
1040 return rtp_params;
1041 }
1042
SetRtpSendParameters(uint32_t ssrc,const webrtc::RtpParameters & parameters,webrtc::SetParametersCallback callback)1043 webrtc::RTCError WebRtcVideoChannel::SetRtpSendParameters(
1044 uint32_t ssrc,
1045 const webrtc::RtpParameters& parameters,
1046 webrtc::SetParametersCallback callback) {
1047 RTC_DCHECK_RUN_ON(&thread_checker_);
1048 TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRtpSendParameters");
1049 auto it = send_streams_.find(ssrc);
1050 if (it == send_streams_.end()) {
1051 RTC_LOG(LS_ERROR) << "Attempting to set RTP send parameters for stream "
1052 "with ssrc "
1053 << ssrc << " which doesn't exist.";
1054 return webrtc::InvokeSetParametersCallback(
1055 callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR));
1056 }
1057
1058 // TODO(deadbeef): Handle setting parameters with a list of codecs in a
1059 // different order (which should change the send codec).
1060 webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc);
1061 if (current_parameters.codecs != parameters.codecs) {
1062 RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs "
1063 "is not currently supported.";
1064 return webrtc::InvokeSetParametersCallback(
1065 callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR));
1066 }
1067
1068 if (!parameters.encodings.empty()) {
1069 // Note that these values come from:
1070 // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5
1071 // TODO(deadbeef): Change values depending on whether we are sending a
1072 // keyframe or non-keyframe.
1073 rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT;
1074 switch (parameters.encodings[0].network_priority) {
1075 case webrtc::Priority::kVeryLow:
1076 new_dscp = rtc::DSCP_CS1;
1077 break;
1078 case webrtc::Priority::kLow:
1079 new_dscp = rtc::DSCP_DEFAULT;
1080 break;
1081 case webrtc::Priority::kMedium:
1082 new_dscp = rtc::DSCP_AF42;
1083 break;
1084 case webrtc::Priority::kHigh:
1085 new_dscp = rtc::DSCP_AF41;
1086 break;
1087 }
1088 SetPreferredDscp(new_dscp);
1089 }
1090
1091 return it->second->SetRtpParameters(parameters, std::move(callback));
1092 }
1093
GetRtpReceiveParameters(uint32_t ssrc) const1094 webrtc::RtpParameters WebRtcVideoChannel::GetRtpReceiveParameters(
1095 uint32_t ssrc) const {
1096 RTC_DCHECK_RUN_ON(&thread_checker_);
1097 webrtc::RtpParameters rtp_params;
1098 auto it = receive_streams_.find(ssrc);
1099 if (it == receive_streams_.end()) {
1100 RTC_LOG(LS_WARNING)
1101 << "Attempting to get RTP receive parameters for stream "
1102 "with SSRC "
1103 << ssrc << " which doesn't exist.";
1104 return webrtc::RtpParameters();
1105 }
1106 rtp_params = it->second->GetRtpParameters();
1107
1108 // Add codecs, which any stream is prepared to receive.
1109 for (const VideoCodec& codec : recv_params_.codecs) {
1110 rtp_params.codecs.push_back(codec.ToCodecParameters());
1111 }
1112
1113 return rtp_params;
1114 }
1115
GetDefaultRtpReceiveParameters() const1116 webrtc::RtpParameters WebRtcVideoChannel::GetDefaultRtpReceiveParameters()
1117 const {
1118 RTC_DCHECK_RUN_ON(&thread_checker_);
1119 webrtc::RtpParameters rtp_params;
1120 if (!default_unsignalled_ssrc_handler_.GetDefaultSink()) {
1121 // Getting parameters on a default, unsignaled video receive stream but
1122 // because we've not configured to receive such a stream, `encodings` is
1123 // empty.
1124 return rtp_params;
1125 }
1126 rtp_params.encodings.emplace_back();
1127
1128 // Add codecs, which any stream is prepared to receive.
1129 for (const VideoCodec& codec : recv_params_.codecs) {
1130 rtp_params.codecs.push_back(codec.ToCodecParameters());
1131 }
1132
1133 return rtp_params;
1134 }
1135
GetChangedRecvParameters(const VideoRecvParameters & params,ChangedRecvParameters * changed_params) const1136 bool WebRtcVideoChannel::GetChangedRecvParameters(
1137 const VideoRecvParameters& params,
1138 ChangedRecvParameters* changed_params) const {
1139 if (!ValidateCodecFormats(params.codecs) ||
1140 !ValidateRtpExtensions(params.extensions, recv_rtp_extensions_)) {
1141 return false;
1142 }
1143
1144 // Handle receive codecs.
1145 const std::vector<VideoCodecSettings> mapped_codecs =
1146 MapCodecs(params.codecs);
1147 if (mapped_codecs.empty()) {
1148 RTC_LOG(LS_ERROR)
1149 << "GetChangedRecvParameters called without any video codecs.";
1150 return false;
1151 }
1152
1153 // Verify that every mapped codec is supported locally.
1154 if (params.is_stream_active) {
1155 const std::vector<VideoCodec> local_supported_codecs =
1156 GetPayloadTypesAndDefaultCodecs(decoder_factory_,
1157 /*is_decoder_factory=*/true,
1158 /*include_rtx=*/true, call_->trials());
1159 for (const VideoCodecSettings& mapped_codec : mapped_codecs) {
1160 if (!FindMatchingCodec(local_supported_codecs, mapped_codec.codec)) {
1161 RTC_LOG(LS_ERROR)
1162 << "GetChangedRecvParameters called with unsupported video codec: "
1163 << mapped_codec.codec.ToString();
1164 return false;
1165 }
1166 }
1167 }
1168
1169 if (NonFlexfecReceiveCodecsHaveChanged(recv_codecs_, mapped_codecs)) {
1170 changed_params->codec_settings =
1171 absl::optional<std::vector<VideoCodecSettings>>(mapped_codecs);
1172 }
1173
1174 // Handle RTP header extensions.
1175 std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
1176 params.extensions, webrtc::RtpExtension::IsSupportedForVideo, false,
1177 call_->trials());
1178 if (filtered_extensions != recv_rtp_extensions_) {
1179 changed_params->rtp_header_extensions =
1180 absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
1181 }
1182
1183 int flexfec_payload_type = mapped_codecs.front().flexfec_payload_type;
1184 if (flexfec_payload_type != recv_flexfec_payload_type_) {
1185 changed_params->flexfec_payload_type = flexfec_payload_type;
1186 }
1187
1188 return true;
1189 }
1190
SetRecvParameters(const VideoRecvParameters & params)1191 bool WebRtcVideoChannel::SetRecvParameters(const VideoRecvParameters& params) {
1192 RTC_DCHECK_RUN_ON(&thread_checker_);
1193 TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRecvParameters");
1194 RTC_LOG(LS_INFO) << "SetRecvParameters: " << params.ToString();
1195 ChangedRecvParameters changed_params;
1196 if (!GetChangedRecvParameters(params, &changed_params)) {
1197 return false;
1198 }
1199 if (changed_params.flexfec_payload_type) {
1200 RTC_DLOG(LS_INFO) << "Changing FlexFEC payload type (recv) from "
1201 << recv_flexfec_payload_type_ << " to "
1202 << *changed_params.flexfec_payload_type;
1203 recv_flexfec_payload_type_ = *changed_params.flexfec_payload_type;
1204 }
1205 if (changed_params.rtp_header_extensions) {
1206 recv_rtp_extensions_ = *changed_params.rtp_header_extensions;
1207 }
1208 if (changed_params.codec_settings) {
1209 RTC_DLOG(LS_INFO) << "Changing recv codecs from "
1210 << CodecSettingsVectorToString(recv_codecs_) << " to "
1211 << CodecSettingsVectorToString(
1212 *changed_params.codec_settings);
1213 recv_codecs_ = *changed_params.codec_settings;
1214 }
1215
1216 for (auto& kv : receive_streams_) {
1217 kv.second->SetRecvParameters(changed_params);
1218 }
1219 recv_params_ = params;
1220 return true;
1221 }
1222
CodecSettingsVectorToString(const std::vector<VideoCodecSettings> & codecs)1223 std::string WebRtcVideoChannel::CodecSettingsVectorToString(
1224 const std::vector<VideoCodecSettings>& codecs) {
1225 rtc::StringBuilder out;
1226 out << "{";
1227 for (size_t i = 0; i < codecs.size(); ++i) {
1228 out << codecs[i].codec.ToString();
1229 if (i != codecs.size() - 1) {
1230 out << ", ";
1231 }
1232 }
1233 out << "}";
1234 return out.Release();
1235 }
1236
ExtractCodecInformation(rtc::ArrayView<const VideoCodecSettings> recv_codecs,std::map<int,int> & rtx_associated_payload_types,std::set<int> & raw_payload_types,std::vector<webrtc::VideoReceiveStreamInterface::Decoder> & decoders)1237 void WebRtcVideoChannel::ExtractCodecInformation(
1238 rtc::ArrayView<const VideoCodecSettings> recv_codecs,
1239 std::map<int, int>& rtx_associated_payload_types,
1240 std::set<int>& raw_payload_types,
1241 std::vector<webrtc::VideoReceiveStreamInterface::Decoder>& decoders) {
1242 RTC_DCHECK(!recv_codecs.empty());
1243 RTC_DCHECK(rtx_associated_payload_types.empty());
1244 RTC_DCHECK(raw_payload_types.empty());
1245 RTC_DCHECK(decoders.empty());
1246
1247 for (const VideoCodecSettings& recv_codec : recv_codecs) {
1248 decoders.emplace_back(
1249 webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params),
1250 recv_codec.codec.id);
1251 rtx_associated_payload_types.emplace(recv_codec.rtx_payload_type,
1252 recv_codec.codec.id);
1253 if (recv_codec.codec.packetization == kPacketizationParamRaw) {
1254 raw_payload_types.insert(recv_codec.codec.id);
1255 }
1256 }
1257 }
1258
SetReceiverReportSsrc(uint32_t ssrc)1259 void WebRtcVideoChannel::SetReceiverReportSsrc(uint32_t ssrc) {
1260 RTC_DCHECK_RUN_ON(&thread_checker_);
1261 if (ssrc == rtcp_receiver_report_ssrc_)
1262 return;
1263
1264 rtcp_receiver_report_ssrc_ = ssrc;
1265 for (auto& [unused, receive_stream] : receive_streams_)
1266 receive_stream->SetLocalSsrc(ssrc);
1267 }
1268
GetSendCodec(VideoCodec * codec)1269 bool WebRtcVideoChannel::GetSendCodec(VideoCodec* codec) {
1270 RTC_DCHECK_RUN_ON(&thread_checker_);
1271 if (!send_codec_) {
1272 RTC_LOG(LS_VERBOSE) << "GetSendCodec: No send codec set.";
1273 return false;
1274 }
1275 *codec = send_codec_->codec;
1276 return true;
1277 }
1278
SetSend(bool send)1279 bool WebRtcVideoChannel::SetSend(bool send) {
1280 RTC_DCHECK_RUN_ON(&thread_checker_);
1281 TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSend");
1282 RTC_LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false");
1283 if (send && !send_codec_) {
1284 RTC_DLOG(LS_ERROR) << "SetSend(true) called before setting codec.";
1285 return false;
1286 }
1287 for (const auto& kv : send_streams_) {
1288 kv.second->SetSend(send);
1289 }
1290 sending_ = send;
1291 return true;
1292 }
1293
SetVideoSend(uint32_t ssrc,const VideoOptions * options,rtc::VideoSourceInterface<webrtc::VideoFrame> * source)1294 bool WebRtcVideoChannel::SetVideoSend(
1295 uint32_t ssrc,
1296 const VideoOptions* options,
1297 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
1298 RTC_DCHECK_RUN_ON(&thread_checker_);
1299 TRACE_EVENT0("webrtc", "SetVideoSend");
1300 RTC_DCHECK(ssrc != 0);
1301 RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: "
1302 << (options ? options->ToString() : "nullptr")
1303 << ", source = " << (source ? "(source)" : "nullptr") << ")";
1304
1305 const auto& kv = send_streams_.find(ssrc);
1306 if (kv == send_streams_.end()) {
1307 // Allow unknown ssrc only if source is null.
1308 RTC_CHECK(source == nullptr);
1309 RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc;
1310 return false;
1311 }
1312
1313 return kv->second->SetVideoSend(options, source);
1314 }
1315
ValidateSendSsrcAvailability(const StreamParams & sp) const1316 bool WebRtcVideoChannel::ValidateSendSsrcAvailability(
1317 const StreamParams& sp) const {
1318 for (uint32_t ssrc : sp.ssrcs) {
1319 if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) {
1320 RTC_LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc
1321 << "' already exists.";
1322 return false;
1323 }
1324 }
1325 return true;
1326 }
1327
ValidateReceiveSsrcAvailability(const StreamParams & sp) const1328 bool WebRtcVideoChannel::ValidateReceiveSsrcAvailability(
1329 const StreamParams& sp) const {
1330 for (uint32_t ssrc : sp.ssrcs) {
1331 if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) {
1332 RTC_LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc
1333 << "' already exists.";
1334 return false;
1335 }
1336 }
1337 return true;
1338 }
1339
AddSendStream(const StreamParams & sp)1340 bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) {
1341 RTC_DCHECK_RUN_ON(&thread_checker_);
1342 RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString();
1343 if (!ValidateStreamParams(sp))
1344 return false;
1345
1346 if (!ValidateSendSsrcAvailability(sp))
1347 return false;
1348
1349 for (uint32_t used_ssrc : sp.ssrcs)
1350 send_ssrcs_.insert(used_ssrc);
1351
1352 webrtc::VideoSendStream::Config config(this);
1353
1354 for (const RidDescription& rid : sp.rids()) {
1355 config.rtp.rids.push_back(rid.rid);
1356 }
1357
1358 config.suspend_below_min_bitrate = video_config_.suspend_below_min_bitrate;
1359 config.periodic_alr_bandwidth_probing =
1360 video_config_.periodic_alr_bandwidth_probing;
1361 config.encoder_settings.experiment_cpu_load_estimator =
1362 video_config_.experiment_cpu_load_estimator;
1363 config.encoder_settings.encoder_factory = encoder_factory_;
1364 config.encoder_settings.bitrate_allocator_factory =
1365 bitrate_allocator_factory_;
1366 config.encoder_settings.encoder_switch_request_callback = this;
1367 config.crypto_options = crypto_options_;
1368 config.rtp.extmap_allow_mixed = ExtmapAllowMixed();
1369 config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms;
1370
1371 WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
1372 call_, sp, std::move(config), default_send_options_,
1373 video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps,
1374 send_codec_, send_rtp_extensions_, send_params_);
1375
1376 uint32_t ssrc = sp.first_ssrc();
1377 RTC_DCHECK(ssrc != 0);
1378 send_streams_[ssrc] = stream;
1379
1380 if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
1381 SetReceiverReportSsrc(ssrc);
1382 }
1383
1384 if (sending_) {
1385 stream->SetSend(true);
1386 }
1387
1388 return true;
1389 }
1390
RemoveSendStream(uint32_t ssrc)1391 bool WebRtcVideoChannel::RemoveSendStream(uint32_t ssrc) {
1392 RTC_DCHECK_RUN_ON(&thread_checker_);
1393 RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc;
1394
1395 WebRtcVideoSendStream* removed_stream;
1396 std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
1397 send_streams_.find(ssrc);
1398 if (it == send_streams_.end()) {
1399 return false;
1400 }
1401
1402 for (uint32_t old_ssrc : it->second->GetSsrcs())
1403 send_ssrcs_.erase(old_ssrc);
1404
1405 removed_stream = it->second;
1406 send_streams_.erase(it);
1407
1408 // Switch receiver report SSRCs, the one in use is no longer valid.
1409 if (rtcp_receiver_report_ssrc_ == ssrc) {
1410 SetReceiverReportSsrc(send_streams_.empty() ? kDefaultRtcpReceiverReportSsrc
1411 : send_streams_.begin()->first);
1412 }
1413
1414 delete removed_stream;
1415
1416 return true;
1417 }
1418
DeleteReceiveStream(WebRtcVideoChannel::WebRtcVideoReceiveStream * stream)1419 void WebRtcVideoChannel::DeleteReceiveStream(
1420 WebRtcVideoChannel::WebRtcVideoReceiveStream* stream) {
1421 for (uint32_t old_ssrc : stream->GetSsrcs())
1422 receive_ssrcs_.erase(old_ssrc);
1423 delete stream;
1424 }
1425
AddRecvStream(const StreamParams & sp)1426 bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp) {
1427 return AddRecvStream(sp, false);
1428 }
1429
AddRecvStream(const StreamParams & sp,bool default_stream)1430 bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp,
1431 bool default_stream) {
1432 RTC_DCHECK_RUN_ON(&thread_checker_);
1433
1434 RTC_LOG(LS_INFO) << "AddRecvStream"
1435 << (default_stream ? " (default stream)" : "") << ": "
1436 << sp.ToString();
1437 if (!sp.has_ssrcs()) {
1438 // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used
1439 // later when we know the SSRC on the first packet arrival.
1440 unsignaled_stream_params_ = sp;
1441 return true;
1442 }
1443
1444 if (!ValidateStreamParams(sp))
1445 return false;
1446
1447 for (uint32_t ssrc : sp.ssrcs) {
1448 // Remove running stream if this was a default stream.
1449 const auto& prev_stream = receive_streams_.find(ssrc);
1450 if (prev_stream != receive_streams_.end()) {
1451 if (default_stream || !prev_stream->second->IsDefaultStream()) {
1452 RTC_LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc
1453 << "' already exists.";
1454 return false;
1455 }
1456 DeleteReceiveStream(prev_stream->second);
1457 receive_streams_.erase(prev_stream);
1458 }
1459 }
1460
1461 if (!ValidateReceiveSsrcAvailability(sp))
1462 return false;
1463
1464 for (uint32_t used_ssrc : sp.ssrcs)
1465 receive_ssrcs_.insert(used_ssrc);
1466
1467 webrtc::VideoReceiveStreamInterface::Config config(this, decoder_factory_);
1468 webrtc::FlexfecReceiveStream::Config flexfec_config(this);
1469 ConfigureReceiverRtp(&config, &flexfec_config, sp);
1470
1471 config.crypto_options = crypto_options_;
1472 config.enable_prerenderer_smoothing =
1473 video_config_.enable_prerenderer_smoothing;
1474 if (!sp.stream_ids().empty()) {
1475 config.sync_group = sp.stream_ids()[0];
1476 }
1477
1478 if (unsignaled_frame_transformer_ && !config.frame_transformer)
1479 config.frame_transformer = unsignaled_frame_transformer_;
1480
1481 receive_streams_[sp.first_ssrc()] = new WebRtcVideoReceiveStream(
1482 this, call_, sp, std::move(config), default_stream, recv_codecs_,
1483 flexfec_config);
1484
1485 return true;
1486 }
1487
ConfigureReceiverRtp(webrtc::VideoReceiveStreamInterface::Config * config,webrtc::FlexfecReceiveStream::Config * flexfec_config,const StreamParams & sp) const1488 void WebRtcVideoChannel::ConfigureReceiverRtp(
1489 webrtc::VideoReceiveStreamInterface::Config* config,
1490 webrtc::FlexfecReceiveStream::Config* flexfec_config,
1491 const StreamParams& sp) const {
1492 uint32_t ssrc = sp.first_ssrc();
1493
1494 config->rtp.remote_ssrc = ssrc;
1495 config->rtp.local_ssrc = rtcp_receiver_report_ssrc_;
1496
1497 // TODO(pbos): This protection is against setting the same local ssrc as
1498 // remote which is not permitted by the lower-level API. RTCP requires a
1499 // corresponding sender SSRC. Figure out what to do when we don't have
1500 // (receive-only) or know a good local SSRC.
1501 if (config->rtp.remote_ssrc == config->rtp.local_ssrc) {
1502 if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) {
1503 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc;
1504 } else {
1505 config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1;
1506 }
1507 }
1508
1509 // Whether or not the receive stream sends reduced size RTCP is determined
1510 // by the send params.
1511 // TODO(deadbeef): Once we change "send_params" to "sender_params" and
1512 // "recv_params" to "receiver_params", we should get this out of
1513 // receiver_params_.
1514 config->rtp.rtcp_mode = send_params_.rtcp.reduced_size
1515 ? webrtc::RtcpMode::kReducedSize
1516 : webrtc::RtcpMode::kCompound;
1517
1518 // rtx-time (RFC 4588) is a declarative attribute similar to rtcp-rsize and
1519 // determined by the sender / send codec.
1520 if (send_codec_ && send_codec_->rtx_time != -1) {
1521 config->rtp.nack.rtp_history_ms = send_codec_->rtx_time;
1522 }
1523
1524 config->rtp.transport_cc =
1525 send_codec_ ? HasTransportCc(send_codec_->codec) : false;
1526
1527 sp.GetFidSsrc(ssrc, &config->rtp.rtx_ssrc);
1528
1529 config->rtp.extensions = recv_rtp_extensions_;
1530
1531 // TODO(brandtr): Generalize when we add support for multistream protection.
1532 flexfec_config->payload_type = recv_flexfec_payload_type_;
1533 if (!IsDisabled(call_->trials(), "WebRTC-FlexFEC-03-Advertised") &&
1534 sp.GetFecFrSsrc(ssrc, &flexfec_config->rtp.remote_ssrc)) {
1535 flexfec_config->protected_media_ssrcs = {ssrc};
1536 flexfec_config->rtp.local_ssrc = config->rtp.local_ssrc;
1537 flexfec_config->rtcp_mode = config->rtp.rtcp_mode;
1538 // TODO(brandtr): We should be spec-compliant and set `transport_cc` here
1539 // based on the rtcp-fb for the FlexFEC codec, not the media codec.
1540 flexfec_config->rtp.transport_cc = config->rtp.transport_cc;
1541 flexfec_config->rtp.extensions = config->rtp.extensions;
1542 }
1543 }
1544
RemoveRecvStream(uint32_t ssrc)1545 bool WebRtcVideoChannel::RemoveRecvStream(uint32_t ssrc) {
1546 RTC_DCHECK_RUN_ON(&thread_checker_);
1547 RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc;
1548
1549 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream =
1550 receive_streams_.find(ssrc);
1551 if (stream == receive_streams_.end()) {
1552 RTC_LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc;
1553 return false;
1554 }
1555 DeleteReceiveStream(stream->second);
1556 receive_streams_.erase(stream);
1557
1558 return true;
1559 }
1560
ResetUnsignaledRecvStream()1561 void WebRtcVideoChannel::ResetUnsignaledRecvStream() {
1562 RTC_DCHECK_RUN_ON(&thread_checker_);
1563 RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream.";
1564 unsignaled_stream_params_ = StreamParams();
1565 last_unsignalled_ssrc_creation_time_ms_ = absl::nullopt;
1566
1567 // Delete any created default streams. This is needed to avoid SSRC collisions
1568 // in Call's RtpDemuxer, in the case that `this` has created a default video
1569 // receiver, and then some other WebRtcVideoChannel gets the SSRC signaled
1570 // in the corresponding Unified Plan "m=" section.
1571 auto it = receive_streams_.begin();
1572 while (it != receive_streams_.end()) {
1573 if (it->second->IsDefaultStream()) {
1574 DeleteReceiveStream(it->second);
1575 receive_streams_.erase(it++);
1576 } else {
1577 ++it;
1578 }
1579 }
1580 }
1581
OnDemuxerCriteriaUpdatePending()1582 void WebRtcVideoChannel::OnDemuxerCriteriaUpdatePending() {
1583 RTC_DCHECK_RUN_ON(&thread_checker_);
1584 ++demuxer_criteria_id_;
1585 }
1586
OnDemuxerCriteriaUpdateComplete()1587 void WebRtcVideoChannel::OnDemuxerCriteriaUpdateComplete() {
1588 RTC_DCHECK_RUN_ON(&thread_checker_);
1589 ++demuxer_criteria_completed_id_;
1590 }
1591
SetSink(uint32_t ssrc,rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)1592 bool WebRtcVideoChannel::SetSink(
1593 uint32_t ssrc,
1594 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
1595 RTC_DCHECK_RUN_ON(&thread_checker_);
1596 RTC_LOG(LS_INFO) << "SetSink: ssrc:" << ssrc << " "
1597 << (sink ? "(ptr)" : "nullptr");
1598
1599 std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
1600 receive_streams_.find(ssrc);
1601 if (it == receive_streams_.end()) {
1602 return false;
1603 }
1604
1605 it->second->SetSink(sink);
1606 return true;
1607 }
1608
SetDefaultSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)1609 void WebRtcVideoChannel::SetDefaultSink(
1610 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
1611 RTC_DCHECK_RUN_ON(&thread_checker_);
1612 RTC_LOG(LS_INFO) << "SetDefaultSink: " << (sink ? "(ptr)" : "nullptr");
1613 default_unsignalled_ssrc_handler_.SetDefaultSink(this, sink);
1614 }
1615
GetStats(VideoMediaInfo * info)1616 bool WebRtcVideoChannel::GetStats(VideoMediaInfo* info) {
1617 RTC_DCHECK_RUN_ON(&thread_checker_);
1618 TRACE_EVENT0("webrtc", "WebRtcVideoChannel::GetStats");
1619
1620 // Log stats periodically.
1621 bool log_stats = false;
1622 int64_t now_ms = rtc::TimeMillis();
1623 if (last_stats_log_ms_ == -1 ||
1624 now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
1625 last_stats_log_ms_ = now_ms;
1626 log_stats = true;
1627 }
1628
1629 info->Clear();
1630 FillSenderStats(info, log_stats);
1631 FillReceiverStats(info, log_stats);
1632 FillSendAndReceiveCodecStats(info);
1633 // TODO(holmer): We should either have rtt available as a metric on
1634 // VideoSend/ReceiveStreams, or we should remove rtt from VideoSenderInfo.
1635 webrtc::Call::Stats stats = call_->GetStats();
1636 if (stats.rtt_ms != -1) {
1637 for (size_t i = 0; i < info->senders.size(); ++i) {
1638 info->senders[i].rtt_ms = stats.rtt_ms;
1639 }
1640 for (size_t i = 0; i < info->aggregated_senders.size(); ++i) {
1641 info->aggregated_senders[i].rtt_ms = stats.rtt_ms;
1642 }
1643 }
1644
1645 if (log_stats)
1646 RTC_LOG(LS_INFO) << stats.ToString(now_ms);
1647
1648 return true;
1649 }
1650
FillSenderStats(VideoMediaInfo * video_media_info,bool log_stats)1651 void WebRtcVideoChannel::FillSenderStats(VideoMediaInfo* video_media_info,
1652 bool log_stats) {
1653 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
1654 send_streams_.begin();
1655 it != send_streams_.end(); ++it) {
1656 auto infos = it->second->GetPerLayerVideoSenderInfos(log_stats);
1657 if (infos.empty())
1658 continue;
1659 video_media_info->aggregated_senders.push_back(
1660 it->second->GetAggregatedVideoSenderInfo(infos));
1661 for (auto&& info : infos) {
1662 video_media_info->senders.push_back(info);
1663 }
1664 }
1665 }
1666
FillReceiverStats(VideoMediaInfo * video_media_info,bool log_stats)1667 void WebRtcVideoChannel::FillReceiverStats(VideoMediaInfo* video_media_info,
1668 bool log_stats) {
1669 for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
1670 receive_streams_.begin();
1671 it != receive_streams_.end(); ++it) {
1672 video_media_info->receivers.push_back(
1673 it->second->GetVideoReceiverInfo(log_stats));
1674 }
1675 }
1676
FillBitrateInfo(BandwidthEstimationInfo * bwe_info)1677 void WebRtcVideoChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) {
1678 RTC_DCHECK_RUN_ON(&thread_checker_);
1679 for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream =
1680 send_streams_.begin();
1681 stream != send_streams_.end(); ++stream) {
1682 stream->second->FillBitrateInfo(bwe_info);
1683 }
1684 }
1685
FillSendAndReceiveCodecStats(VideoMediaInfo * video_media_info)1686 void WebRtcVideoChannel::FillSendAndReceiveCodecStats(
1687 VideoMediaInfo* video_media_info) {
1688 for (const VideoCodec& codec : send_params_.codecs) {
1689 webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
1690 video_media_info->send_codecs.insert(
1691 std::make_pair(codec_params.payload_type, std::move(codec_params)));
1692 }
1693 for (const VideoCodec& codec : recv_params_.codecs) {
1694 webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
1695 video_media_info->receive_codecs.insert(
1696 std::make_pair(codec_params.payload_type, std::move(codec_params)));
1697 }
1698 }
1699
OnPacketReceived(rtc::CopyOnWriteBuffer packet,int64_t packet_time_us)1700 void WebRtcVideoChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet,
1701 int64_t packet_time_us) {
1702 RTC_DCHECK_RUN_ON(&network_thread_checker_);
1703 // TODO(bugs.webrtc.org/11993): This code is very similar to what
1704 // WebRtcVoiceMediaChannel::OnPacketReceived does. For maintainability and
1705 // consistency it would be good to move the interaction with call_->Receiver()
1706 // to a common implementation and provide a callback on the worker thread
1707 // for the exception case (DELIVERY_UNKNOWN_SSRC) and how retry is attempted.
1708 worker_thread_->PostTask(
1709 SafeTask(task_safety_.flag(), [this, packet, packet_time_us] {
1710 RTC_DCHECK_RUN_ON(&thread_checker_);
1711 const webrtc::PacketReceiver::DeliveryStatus delivery_result =
1712 call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1713 packet_time_us);
1714 switch (delivery_result) {
1715 case webrtc::PacketReceiver::DELIVERY_OK:
1716 return;
1717 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
1718 return;
1719 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
1720 break;
1721 }
1722
1723 absl::optional<uint32_t> rtx_ssrc;
1724 uint32_t ssrc = ParseRtpSsrc(packet);
1725
1726 if (unknown_ssrc_packet_buffer_) {
1727 unknown_ssrc_packet_buffer_->AddPacket(ssrc, packet_time_us, packet);
1728 return;
1729 }
1730
1731 if (discard_unknown_ssrc_packets_) {
1732 return;
1733 }
1734
1735 int payload_type = ParseRtpPayloadType(packet);
1736
1737 // See if this payload_type is registered as one that usually gets its
1738 // own SSRC (RTX) or at least is safe to drop either way (FEC). If it
1739 // is, and it wasn't handled above by DeliverPacket, that means we don't
1740 // know what stream it associates with, and we shouldn't ever create an
1741 // implicit channel for these.
1742 for (auto& codec : recv_codecs_) {
1743 if (payload_type == codec.ulpfec.red_rtx_payload_type ||
1744 payload_type == codec.ulpfec.ulpfec_payload_type) {
1745 return;
1746 }
1747 if (payload_type == codec.rtx_payload_type) {
1748 // As we don't support receiving simulcast there can only be one RTX
1749 // stream, which will be associated with unsignaled media stream.
1750 // It is not possible to update the ssrcs of a receive stream, so we
1751 // recreate it insead if found.
1752 auto default_ssrc = GetDefaultReceiveStreamSsrc();
1753 if (!default_ssrc) {
1754 return;
1755 }
1756 rtx_ssrc = ssrc;
1757 ssrc = *default_ssrc;
1758 // Allow recreating the receive stream even if the RTX packet is
1759 // received just after the media packet.
1760 last_unsignalled_ssrc_creation_time_ms_.reset();
1761 break;
1762 }
1763 }
1764 if (payload_type == recv_flexfec_payload_type_) {
1765 return;
1766 }
1767
1768 // Ignore unknown ssrcs if there is a demuxer criteria update pending.
1769 // During a demuxer update we may receive ssrcs that were recently
1770 // removed or we may receve ssrcs that were recently configured for a
1771 // different video channel.
1772 if (demuxer_criteria_id_ != demuxer_criteria_completed_id_) {
1773 return;
1774 }
1775 // Ignore unknown ssrcs if we recently created an unsignalled receive
1776 // stream since this shouldn't happen frequently. Getting into a state
1777 // of creating decoders on every packet eats up processing time (e.g.
1778 // https://crbug.com/1069603) and this cooldown prevents that.
1779 if (last_unsignalled_ssrc_creation_time_ms_.has_value()) {
1780 int64_t now_ms = rtc::TimeMillis();
1781 if (now_ms - last_unsignalled_ssrc_creation_time_ms_.value() <
1782 kUnsignaledSsrcCooldownMs) {
1783 // We've already created an unsignalled ssrc stream within the last
1784 // 0.5 s, ignore with a warning.
1785 RTC_LOG(LS_WARNING)
1786 << "Another unsignalled ssrc packet arrived shortly after the "
1787 << "creation of an unsignalled ssrc stream. Dropping packet.";
1788 return;
1789 }
1790 }
1791 // Let the unsignalled ssrc handler decide whether to drop or deliver.
1792 switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc,
1793 rtx_ssrc)) {
1794 case UnsignalledSsrcHandler::kDropPacket:
1795 return;
1796 case UnsignalledSsrcHandler::kDeliverPacket:
1797 break;
1798 }
1799
1800 if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1801 packet_time_us) !=
1802 webrtc::PacketReceiver::DELIVERY_OK) {
1803 RTC_LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery.";
1804 }
1805 last_unsignalled_ssrc_creation_time_ms_ = rtc::TimeMillis();
1806 }));
1807 }
1808
OnPacketSent(const rtc::SentPacket & sent_packet)1809 void WebRtcVideoChannel::OnPacketSent(const rtc::SentPacket& sent_packet) {
1810 RTC_DCHECK_RUN_ON(&network_thread_checker_);
1811 // TODO(tommi): We shouldn't need to go through call_ to deliver this
1812 // notification. We should already have direct access to
1813 // video_send_delay_stats_ and transport_send_ptr_ via `stream_`.
1814 // So we should be able to remove OnSentPacket from Call and handle this per
1815 // channel instead. At the moment Call::OnSentPacket calls OnSentPacket for
1816 // the video stats, for all sent packets, including audio, which causes
1817 // unnecessary lookups.
1818 call_->OnSentPacket(sent_packet);
1819 }
1820
BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs)1821 void WebRtcVideoChannel::BackfillBufferedPackets(
1822 rtc::ArrayView<const uint32_t> ssrcs) {
1823 RTC_DCHECK_RUN_ON(&thread_checker_);
1824 if (!unknown_ssrc_packet_buffer_) {
1825 return;
1826 }
1827
1828 int delivery_ok_cnt = 0;
1829 int delivery_unknown_ssrc_cnt = 0;
1830 int delivery_packet_error_cnt = 0;
1831 webrtc::PacketReceiver* receiver = this->call_->Receiver();
1832 unknown_ssrc_packet_buffer_->BackfillPackets(
1833 ssrcs, [&](uint32_t /*ssrc*/, int64_t packet_time_us,
1834 rtc::CopyOnWriteBuffer packet) {
1835 switch (receiver->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1836 packet_time_us)) {
1837 case webrtc::PacketReceiver::DELIVERY_OK:
1838 delivery_ok_cnt++;
1839 break;
1840 case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
1841 delivery_unknown_ssrc_cnt++;
1842 break;
1843 case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
1844 delivery_packet_error_cnt++;
1845 break;
1846 }
1847 });
1848 rtc::StringBuilder out;
1849 out << "[ ";
1850 for (uint32_t ssrc : ssrcs) {
1851 out << std::to_string(ssrc) << " ";
1852 }
1853 out << "]";
1854 auto level = rtc::LS_INFO;
1855 if (delivery_unknown_ssrc_cnt > 0 || delivery_packet_error_cnt > 0) {
1856 level = rtc::LS_ERROR;
1857 }
1858 int total =
1859 delivery_ok_cnt + delivery_unknown_ssrc_cnt + delivery_packet_error_cnt;
1860 RTC_LOG_V(level) << "Backfilled " << total
1861 << " packets for ssrcs: " << out.Release()
1862 << " ok: " << delivery_ok_cnt
1863 << " error: " << delivery_packet_error_cnt
1864 << " unknown: " << delivery_unknown_ssrc_cnt;
1865 }
1866
OnReadyToSend(bool ready)1867 void WebRtcVideoChannel::OnReadyToSend(bool ready) {
1868 RTC_DCHECK_RUN_ON(&network_thread_checker_);
1869 RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready.");
1870 call_->SignalChannelNetworkState(
1871 webrtc::MediaType::VIDEO,
1872 ready ? webrtc::kNetworkUp : webrtc::kNetworkDown);
1873 }
1874
OnNetworkRouteChanged(absl::string_view transport_name,const rtc::NetworkRoute & network_route)1875 void WebRtcVideoChannel::OnNetworkRouteChanged(
1876 absl::string_view transport_name,
1877 const rtc::NetworkRoute& network_route) {
1878 RTC_DCHECK_RUN_ON(&network_thread_checker_);
1879 worker_thread_->PostTask(SafeTask(
1880 task_safety_.flag(),
1881 [this, name = std::string(transport_name), route = network_route] {
1882 RTC_DCHECK_RUN_ON(&thread_checker_);
1883 webrtc::RtpTransportControllerSendInterface* transport =
1884 call_->GetTransportControllerSend();
1885 transport->OnNetworkRouteChanged(name, route);
1886 transport->OnTransportOverheadChanged(route.packet_overhead);
1887 }));
1888 }
1889
SetInterface(NetworkInterface * iface)1890 void WebRtcVideoChannel::SetInterface(NetworkInterface* iface) {
1891 RTC_DCHECK_RUN_ON(&network_thread_checker_);
1892 MediaChannel::SetInterface(iface);
1893 // Set the RTP recv/send buffer to a bigger size.
1894 MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF,
1895 kVideoRtpRecvBufferSize);
1896
1897 // Speculative change to increase the outbound socket buffer size.
1898 // In b/15152257, we are seeing a significant number of packets discarded
1899 // due to lack of socket buffer space, although it's not yet clear what the
1900 // ideal value should be.
1901 const std::string group_name_send_buf_size =
1902 call_->trials().Lookup("WebRTC-SendBufferSizeBytes");
1903 int send_buffer_size = kVideoRtpSendBufferSize;
1904 if (!group_name_send_buf_size.empty() &&
1905 (sscanf(group_name_send_buf_size.c_str(), "%d", &send_buffer_size) != 1 ||
1906 send_buffer_size <= 0)) {
1907 RTC_LOG(LS_WARNING) << "Invalid send buffer size: "
1908 << group_name_send_buf_size;
1909 send_buffer_size = kVideoRtpSendBufferSize;
1910 }
1911
1912 MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF,
1913 send_buffer_size);
1914 }
1915
SetFrameDecryptor(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)1916 void WebRtcVideoChannel::SetFrameDecryptor(
1917 uint32_t ssrc,
1918 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
1919 RTC_DCHECK_RUN_ON(&thread_checker_);
1920 auto matching_stream = receive_streams_.find(ssrc);
1921 if (matching_stream != receive_streams_.end()) {
1922 matching_stream->second->SetFrameDecryptor(frame_decryptor);
1923 }
1924 }
1925
SetFrameEncryptor(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor)1926 void WebRtcVideoChannel::SetFrameEncryptor(
1927 uint32_t ssrc,
1928 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
1929 RTC_DCHECK_RUN_ON(&thread_checker_);
1930 auto matching_stream = send_streams_.find(ssrc);
1931 if (matching_stream != send_streams_.end()) {
1932 matching_stream->second->SetFrameEncryptor(frame_encryptor);
1933 } else {
1934 RTC_LOG(LS_ERROR) << "No stream found to attach frame encryptor";
1935 }
1936 }
1937
SetEncoderSelector(uint32_t ssrc,webrtc::VideoEncoderFactory::EncoderSelectorInterface * encoder_selector)1938 void WebRtcVideoChannel::SetEncoderSelector(
1939 uint32_t ssrc,
1940 webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
1941 RTC_DCHECK_RUN_ON(&thread_checker_);
1942 auto matching_stream = send_streams_.find(ssrc);
1943 if (matching_stream != send_streams_.end()) {
1944 matching_stream->second->SetEncoderSelector(encoder_selector);
1945 } else {
1946 RTC_LOG(LS_ERROR) << "No stream found to attach encoder selector";
1947 }
1948 }
1949
SetVideoCodecSwitchingEnabled(bool enabled)1950 void WebRtcVideoChannel::SetVideoCodecSwitchingEnabled(bool enabled) {
1951 RTC_DCHECK_RUN_ON(&thread_checker_);
1952 allow_codec_switching_ = enabled;
1953 if (allow_codec_switching_) {
1954 RTC_LOG(LS_INFO) << "Encoder switching enabled.";
1955 }
1956 }
1957
SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,int delay_ms)1958 bool WebRtcVideoChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,
1959 int delay_ms) {
1960 RTC_DCHECK_RUN_ON(&thread_checker_);
1961 absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
1962
1963 // SSRC of 0 represents the default receive stream.
1964 if (ssrc == 0) {
1965 default_recv_base_minimum_delay_ms_ = delay_ms;
1966 }
1967
1968 if (ssrc == 0 && !default_ssrc) {
1969 return true;
1970 }
1971
1972 if (ssrc == 0 && default_ssrc) {
1973 ssrc = default_ssrc.value();
1974 }
1975
1976 auto stream = receive_streams_.find(ssrc);
1977 if (stream != receive_streams_.end()) {
1978 stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms);
1979 return true;
1980 } else {
1981 RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay";
1982 return false;
1983 }
1984 }
1985
GetBaseMinimumPlayoutDelayMs(uint32_t ssrc) const1986 absl::optional<int> WebRtcVideoChannel::GetBaseMinimumPlayoutDelayMs(
1987 uint32_t ssrc) const {
1988 RTC_DCHECK_RUN_ON(&thread_checker_);
1989 // SSRC of 0 represents the default receive stream.
1990 if (ssrc == 0) {
1991 return default_recv_base_minimum_delay_ms_;
1992 }
1993
1994 auto stream = receive_streams_.find(ssrc);
1995 if (stream != receive_streams_.end()) {
1996 return stream->second->GetBaseMinimumPlayoutDelayMs();
1997 } else {
1998 RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay";
1999 return absl::nullopt;
2000 }
2001 }
2002
GetDefaultReceiveStreamSsrc()2003 absl::optional<uint32_t> WebRtcVideoChannel::GetDefaultReceiveStreamSsrc() {
2004 RTC_DCHECK_RUN_ON(&thread_checker_);
2005 absl::optional<uint32_t> ssrc;
2006 for (auto it = receive_streams_.begin(); it != receive_streams_.end(); ++it) {
2007 if (it->second->IsDefaultStream()) {
2008 ssrc.emplace(it->first);
2009 break;
2010 }
2011 }
2012 return ssrc;
2013 }
2014
GetSources(uint32_t ssrc) const2015 std::vector<webrtc::RtpSource> WebRtcVideoChannel::GetSources(
2016 uint32_t ssrc) const {
2017 RTC_DCHECK_RUN_ON(&thread_checker_);
2018 auto it = receive_streams_.find(ssrc);
2019 if (it == receive_streams_.end()) {
2020 // TODO(bugs.webrtc.org/9781): Investigate standard compliance
2021 // with sources for streams that has been removed.
2022 RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:"
2023 << ssrc << " which doesn't exist.";
2024 return {};
2025 }
2026 return it->second->GetSources();
2027 }
2028
SendRtp(const uint8_t * data,size_t len,const webrtc::PacketOptions & options)2029 bool WebRtcVideoChannel::SendRtp(const uint8_t* data,
2030 size_t len,
2031 const webrtc::PacketOptions& options) {
2032 MediaChannel::SendRtp(data, len, options);
2033 return true;
2034 }
2035
SendRtcp(const uint8_t * data,size_t len)2036 bool WebRtcVideoChannel::SendRtcp(const uint8_t* data, size_t len) {
2037 MediaChannel::SendRtcp(data, len);
2038 return true;
2039 }
2040
2041 WebRtcVideoChannel::WebRtcVideoSendStream::VideoSendStreamParameters::
VideoSendStreamParameters(webrtc::VideoSendStream::Config config,const VideoOptions & options,int max_bitrate_bps,const absl::optional<VideoCodecSettings> & codec_settings)2042 VideoSendStreamParameters(
2043 webrtc::VideoSendStream::Config config,
2044 const VideoOptions& options,
2045 int max_bitrate_bps,
2046 const absl::optional<VideoCodecSettings>& codec_settings)
2047 : config(std::move(config)),
2048 options(options),
2049 max_bitrate_bps(max_bitrate_bps),
2050 conference_mode(false),
2051 codec_settings(codec_settings) {}
2052
WebRtcVideoSendStream(webrtc::Call * call,const StreamParams & sp,webrtc::VideoSendStream::Config config,const VideoOptions & options,bool enable_cpu_overuse_detection,int max_bitrate_bps,const absl::optional<VideoCodecSettings> & codec_settings,const absl::optional<std::vector<webrtc::RtpExtension>> & rtp_extensions,const VideoSendParameters & send_params)2053 WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
2054 webrtc::Call* call,
2055 const StreamParams& sp,
2056 webrtc::VideoSendStream::Config config,
2057 const VideoOptions& options,
2058 bool enable_cpu_overuse_detection,
2059 int max_bitrate_bps,
2060 const absl::optional<VideoCodecSettings>& codec_settings,
2061 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
2062 // TODO(deadbeef): Don't duplicate information between send_params,
2063 // rtp_extensions, options, etc.
2064 const VideoSendParameters& send_params)
2065 : worker_thread_(call->worker_thread()),
2066 ssrcs_(sp.ssrcs),
2067 ssrc_groups_(sp.ssrc_groups),
2068 call_(call),
2069 enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
2070 source_(nullptr),
2071 stream_(nullptr),
2072 parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
2073 rtp_parameters_(CreateRtpParametersWithEncodings(sp)),
2074 sending_(false),
2075 disable_automatic_resize_(
2076 IsEnabled(call->trials(), "WebRTC-Video-DisableAutomaticResize")) {
2077 // Maximum packet size may come in RtpConfig from external transport, for
2078 // example from QuicTransportInterface implementation, so do not exceed
2079 // given max_packet_size.
2080 parameters_.config.rtp.max_packet_size =
2081 std::min<size_t>(parameters_.config.rtp.max_packet_size, kVideoMtu);
2082 parameters_.conference_mode = send_params.conference_mode;
2083
2084 sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs);
2085
2086 // ValidateStreamParams should prevent this from happening.
2087 RTC_CHECK(!parameters_.config.rtp.ssrcs.empty());
2088 rtp_parameters_.encodings[0].ssrc = parameters_.config.rtp.ssrcs[0];
2089
2090 // RTX.
2091 sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
2092 ¶meters_.config.rtp.rtx.ssrcs);
2093
2094 // FlexFEC SSRCs.
2095 // TODO(brandtr): This code needs to be generalized when we add support for
2096 // multistream protection.
2097 if (IsEnabled(call_->trials(), "WebRTC-FlexFEC-03")) {
2098 uint32_t flexfec_ssrc;
2099 bool flexfec_enabled = false;
2100 for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) {
2101 if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) {
2102 if (flexfec_enabled) {
2103 RTC_LOG(LS_INFO)
2104 << "Multiple FlexFEC streams in local SDP, but "
2105 "our implementation only supports a single FlexFEC "
2106 "stream. Will not enable FlexFEC for proposed "
2107 "stream with SSRC: "
2108 << flexfec_ssrc << ".";
2109 continue;
2110 }
2111
2112 flexfec_enabled = true;
2113 parameters_.config.rtp.flexfec.ssrc = flexfec_ssrc;
2114 parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc};
2115 }
2116 }
2117 }
2118
2119 parameters_.config.rtp.c_name = sp.cname;
2120 if (rtp_extensions) {
2121 parameters_.config.rtp.extensions = *rtp_extensions;
2122 rtp_parameters_.header_extensions = *rtp_extensions;
2123 }
2124 parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
2125 ? webrtc::RtcpMode::kReducedSize
2126 : webrtc::RtcpMode::kCompound;
2127 parameters_.config.rtp.mid = send_params.mid;
2128 rtp_parameters_.rtcp.reduced_size = send_params.rtcp.reduced_size;
2129
2130 if (codec_settings) {
2131 SetCodec(*codec_settings);
2132 }
2133 }
2134
~WebRtcVideoSendStream()2135 WebRtcVideoChannel::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
2136 if (stream_ != NULL) {
2137 call_->DestroyVideoSendStream(stream_);
2138 }
2139 }
2140
SetVideoSend(const VideoOptions * options,rtc::VideoSourceInterface<webrtc::VideoFrame> * source)2141 bool WebRtcVideoChannel::WebRtcVideoSendStream::SetVideoSend(
2142 const VideoOptions* options,
2143 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
2144 TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend");
2145 RTC_DCHECK_RUN_ON(&thread_checker_);
2146
2147 if (options) {
2148 VideoOptions old_options = parameters_.options;
2149 parameters_.options.SetAll(*options);
2150 if (parameters_.options.is_screencast.value_or(false) !=
2151 old_options.is_screencast.value_or(false) &&
2152 parameters_.codec_settings) {
2153 // If screen content settings change, we may need to recreate the codec
2154 // instance so that the correct type is used.
2155
2156 SetCodec(*parameters_.codec_settings);
2157 // Mark screenshare parameter as being updated, then test for any other
2158 // changes that may require codec reconfiguration.
2159 old_options.is_screencast = options->is_screencast;
2160 }
2161 if (parameters_.options != old_options) {
2162 ReconfigureEncoder(nullptr);
2163 }
2164 }
2165
2166 if (source_ && stream_) {
2167 stream_->SetSource(nullptr, webrtc::DegradationPreference::DISABLED);
2168 }
2169 // Switch to the new source.
2170 source_ = source;
2171 if (source && stream_) {
2172 stream_->SetSource(source_, GetDegradationPreference());
2173 }
2174 return true;
2175 }
2176
2177 webrtc::DegradationPreference
GetDegradationPreference() const2178 WebRtcVideoChannel::WebRtcVideoSendStream::GetDegradationPreference() const {
2179 // Do not adapt resolution for screen content as this will likely
2180 // result in blurry and unreadable text.
2181 // `this` acts like a VideoSource to make sure SinkWants are handled on the
2182 // correct thread.
2183 if (!enable_cpu_overuse_detection_) {
2184 return webrtc::DegradationPreference::DISABLED;
2185 }
2186
2187 webrtc::DegradationPreference degradation_preference;
2188 if (rtp_parameters_.degradation_preference.has_value()) {
2189 degradation_preference = *rtp_parameters_.degradation_preference;
2190 } else {
2191 if (parameters_.options.content_hint ==
2192 webrtc::VideoTrackInterface::ContentHint::kFluid) {
2193 degradation_preference =
2194 webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
2195 } else if (parameters_.options.is_screencast.value_or(false) ||
2196 parameters_.options.content_hint ==
2197 webrtc::VideoTrackInterface::ContentHint::kDetailed ||
2198 parameters_.options.content_hint ==
2199 webrtc::VideoTrackInterface::ContentHint::kText) {
2200 degradation_preference =
2201 webrtc::DegradationPreference::MAINTAIN_RESOLUTION;
2202 } else if (IsEnabled(call_->trials(), "WebRTC-Video-BalancedDegradation")) {
2203 // Standard wants balanced by default, but it needs to be tuned first.
2204 degradation_preference = webrtc::DegradationPreference::BALANCED;
2205 } else {
2206 // Keep MAINTAIN_FRAMERATE by default until BALANCED has been tuned for
2207 // all codecs and launched.
2208 degradation_preference =
2209 webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
2210 }
2211 }
2212
2213 return degradation_preference;
2214 }
2215
2216 const std::vector<uint32_t>&
GetSsrcs() const2217 WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
2218 return ssrcs_;
2219 }
2220
SetCodec(const VideoCodecSettings & codec_settings)2221 void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
2222 const VideoCodecSettings& codec_settings) {
2223 RTC_DCHECK_RUN_ON(&thread_checker_);
2224 parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
2225 RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
2226
2227 parameters_.config.rtp.payload_name = codec_settings.codec.name;
2228 parameters_.config.rtp.payload_type = codec_settings.codec.id;
2229 parameters_.config.rtp.raw_payload =
2230 codec_settings.codec.packetization == kPacketizationParamRaw;
2231 parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
2232 parameters_.config.rtp.flexfec.payload_type =
2233 codec_settings.flexfec_payload_type;
2234
2235 // Set RTX payload type if RTX is enabled.
2236 if (!parameters_.config.rtp.rtx.ssrcs.empty()) {
2237 if (codec_settings.rtx_payload_type == -1) {
2238 RTC_LOG(LS_WARNING)
2239 << "RTX SSRCs configured but there's no configured RTX "
2240 "payload type. Ignoring.";
2241 parameters_.config.rtp.rtx.ssrcs.clear();
2242 } else {
2243 parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type;
2244 }
2245 }
2246
2247 const bool has_lntf = HasLntf(codec_settings.codec);
2248 parameters_.config.rtp.lntf.enabled = has_lntf;
2249 parameters_.config.encoder_settings.capabilities.loss_notification = has_lntf;
2250
2251 parameters_.config.rtp.nack.rtp_history_ms =
2252 HasNack(codec_settings.codec) ? kNackHistoryMs : 0;
2253
2254 parameters_.codec_settings = codec_settings;
2255
2256 // TODO(bugs.webrtc.org/8830): Avoid recreation, it should be enough to call
2257 // ReconfigureEncoder.
2258 RTC_LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetCodec.";
2259 RecreateWebRtcStream();
2260 }
2261
SetSendParameters(const ChangedSendParameters & params)2262 void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters(
2263 const ChangedSendParameters& params) {
2264 RTC_DCHECK_RUN_ON(&thread_checker_);
2265 // `recreate_stream` means construction-time parameters have changed and the
2266 // sending stream needs to be reset with the new config.
2267 bool recreate_stream = false;
2268 if (params.rtcp_mode) {
2269 parameters_.config.rtp.rtcp_mode = *params.rtcp_mode;
2270 rtp_parameters_.rtcp.reduced_size =
2271 parameters_.config.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
2272 recreate_stream = true;
2273 }
2274 if (params.extmap_allow_mixed) {
2275 parameters_.config.rtp.extmap_allow_mixed = *params.extmap_allow_mixed;
2276 recreate_stream = true;
2277 }
2278 if (params.rtp_header_extensions) {
2279 parameters_.config.rtp.extensions = *params.rtp_header_extensions;
2280 rtp_parameters_.header_extensions = *params.rtp_header_extensions;
2281 recreate_stream = true;
2282 }
2283 if (params.mid) {
2284 parameters_.config.rtp.mid = *params.mid;
2285 recreate_stream = true;
2286 }
2287 if (params.max_bandwidth_bps) {
2288 parameters_.max_bitrate_bps = *params.max_bandwidth_bps;
2289 ReconfigureEncoder(nullptr);
2290 }
2291 if (params.conference_mode) {
2292 parameters_.conference_mode = *params.conference_mode;
2293 }
2294
2295 // Set codecs and options.
2296 if (params.send_codec) {
2297 SetCodec(*params.send_codec);
2298 recreate_stream = false; // SetCodec has already recreated the stream.
2299 } else if (params.conference_mode && parameters_.codec_settings) {
2300 SetCodec(*parameters_.codec_settings);
2301 recreate_stream = false; // SetCodec has already recreated the stream.
2302 }
2303 if (recreate_stream) {
2304 RTC_LOG(LS_INFO)
2305 << "RecreateWebRtcStream (send) because of SetSendParameters";
2306 RecreateWebRtcStream();
2307 }
2308 }
2309
SetRtpParameters(const webrtc::RtpParameters & new_parameters,webrtc::SetParametersCallback callback)2310 webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
2311 const webrtc::RtpParameters& new_parameters,
2312 webrtc::SetParametersCallback callback) {
2313 RTC_DCHECK_RUN_ON(&thread_checker_);
2314 // This is checked higher in the stack (RtpSender), so this is only checking
2315 // for users accessing the private APIs or tests, not specification
2316 // conformance.
2317 // TODO(orphis): Migrate tests to later make this a DCHECK only
2318 webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues(
2319 rtp_parameters_, new_parameters);
2320 if (!error.ok()) {
2321 // Error is propagated to the callback at a higher level
2322 return error;
2323 }
2324
2325 bool new_param = false;
2326 for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
2327 if ((new_parameters.encodings[i].min_bitrate_bps !=
2328 rtp_parameters_.encodings[i].min_bitrate_bps) ||
2329 (new_parameters.encodings[i].max_bitrate_bps !=
2330 rtp_parameters_.encodings[i].max_bitrate_bps) ||
2331 (new_parameters.encodings[i].max_framerate !=
2332 rtp_parameters_.encodings[i].max_framerate) ||
2333 (new_parameters.encodings[i].scale_resolution_down_by !=
2334 rtp_parameters_.encodings[i].scale_resolution_down_by) ||
2335 (new_parameters.encodings[i].num_temporal_layers !=
2336 rtp_parameters_.encodings[i].num_temporal_layers) ||
2337 (new_parameters.encodings[i].requested_resolution !=
2338 rtp_parameters_.encodings[i].requested_resolution)) {
2339 new_param = true;
2340 break;
2341 }
2342 }
2343
2344 bool new_degradation_preference = false;
2345 if (new_parameters.degradation_preference !=
2346 rtp_parameters_.degradation_preference) {
2347 new_degradation_preference = true;
2348 }
2349
2350 // Some fields (e.g. bitrate priority) only need to update the bitrate
2351 // allocator which is updated via ReconfigureEncoder (however, note that the
2352 // actual encoder should only be reconfigured if needed).
2353 bool reconfigure_encoder = new_param ||
2354 (new_parameters.encodings[0].bitrate_priority !=
2355 rtp_parameters_.encodings[0].bitrate_priority) ||
2356 new_parameters.encodings[0].scalability_mode !=
2357 rtp_parameters_.encodings[0].scalability_mode;
2358
2359 // Note that the simulcast encoder adapter relies on the fact that layers
2360 // de/activation triggers encoder reinitialization.
2361 bool new_send_state = false;
2362 for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
2363 bool new_active = IsLayerActive(new_parameters.encodings[i]);
2364 bool old_active = IsLayerActive(rtp_parameters_.encodings[i]);
2365 if (new_active != old_active) {
2366 new_send_state = true;
2367 }
2368 }
2369 rtp_parameters_ = new_parameters;
2370 // Codecs are currently handled at the WebRtcVideoChannel level.
2371 rtp_parameters_.codecs.clear();
2372 if (reconfigure_encoder || new_send_state) {
2373 // Callback responsibility is delegated to ReconfigureEncoder()
2374 ReconfigureEncoder(std::move(callback));
2375 callback = nullptr;
2376 }
2377 if (new_send_state) {
2378 UpdateSendState();
2379 }
2380 if (new_degradation_preference) {
2381 if (source_ && stream_) {
2382 stream_->SetSource(source_, GetDegradationPreference());
2383 }
2384 }
2385 return webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
2386 }
2387
2388 webrtc::RtpParameters
GetRtpParameters() const2389 WebRtcVideoChannel::WebRtcVideoSendStream::GetRtpParameters() const {
2390 RTC_DCHECK_RUN_ON(&thread_checker_);
2391 return rtp_parameters_;
2392 }
2393
SetFrameEncryptor(rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor)2394 void WebRtcVideoChannel::WebRtcVideoSendStream::SetFrameEncryptor(
2395 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
2396 RTC_DCHECK_RUN_ON(&thread_checker_);
2397 parameters_.config.frame_encryptor = frame_encryptor;
2398 if (stream_) {
2399 RTC_LOG(LS_INFO)
2400 << "RecreateWebRtcStream (send) because of SetFrameEncryptor, ssrc="
2401 << parameters_.config.rtp.ssrcs[0];
2402 RecreateWebRtcStream();
2403 }
2404 }
2405
SetEncoderSelector(webrtc::VideoEncoderFactory::EncoderSelectorInterface * encoder_selector)2406 void WebRtcVideoChannel::WebRtcVideoSendStream::SetEncoderSelector(
2407 webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
2408 RTC_DCHECK_RUN_ON(&thread_checker_);
2409 parameters_.config.encoder_selector = encoder_selector;
2410 if (stream_) {
2411 RTC_LOG(LS_INFO)
2412 << "RecreateWebRtcStream (send) because of SetEncoderSelector, ssrc="
2413 << parameters_.config.rtp.ssrcs[0];
2414 RecreateWebRtcStream();
2415 }
2416 }
2417
UpdateSendState()2418 void WebRtcVideoChannel::WebRtcVideoSendStream::UpdateSendState() {
2419 RTC_DCHECK_RUN_ON(&thread_checker_);
2420 if (sending_) {
2421 RTC_DCHECK(stream_ != nullptr);
2422 size_t num_layers = rtp_parameters_.encodings.size();
2423 if (parameters_.encoder_config.number_of_streams == 1) {
2424 // SVC is used. Only one simulcast layer is present.
2425 num_layers = 1;
2426 }
2427 std::vector<bool> active_layers(num_layers);
2428 for (size_t i = 0; i < num_layers; ++i) {
2429 active_layers[i] = IsLayerActive(rtp_parameters_.encodings[i]);
2430 }
2431 if (parameters_.encoder_config.number_of_streams == 1 &&
2432 rtp_parameters_.encodings.size() > 1) {
2433 // SVC is used.
2434 // The only present simulcast layer should be active if any of the
2435 // configured SVC layers is active.
2436 active_layers[0] =
2437 absl::c_any_of(rtp_parameters_.encodings,
2438 [](const auto& encoding) { return encoding.active; });
2439 }
2440 // This updates what simulcast layers are sending, and possibly starts
2441 // or stops the VideoSendStream.
2442 stream_->StartPerRtpStream(active_layers);
2443 } else {
2444 if (stream_ != nullptr) {
2445 stream_->Stop();
2446 }
2447 }
2448 }
2449
2450 webrtc::VideoEncoderConfig
CreateVideoEncoderConfig(const VideoCodec & codec) const2451 WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(
2452 const VideoCodec& codec) const {
2453 RTC_DCHECK_RUN_ON(&thread_checker_);
2454 webrtc::VideoEncoderConfig encoder_config;
2455 encoder_config.codec_type = webrtc::PayloadStringToCodecType(codec.name);
2456 encoder_config.video_format =
2457 webrtc::SdpVideoFormat(codec.name, codec.params);
2458
2459 bool is_screencast = parameters_.options.is_screencast.value_or(false);
2460 if (is_screencast) {
2461 encoder_config.min_transmit_bitrate_bps =
2462 1000 * parameters_.options.screencast_min_bitrate_kbps.value_or(0);
2463 encoder_config.content_type =
2464 webrtc::VideoEncoderConfig::ContentType::kScreen;
2465 } else {
2466 encoder_config.min_transmit_bitrate_bps = 0;
2467 encoder_config.content_type =
2468 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
2469 }
2470
2471 // By default, the stream count for the codec configuration should match the
2472 // number of negotiated ssrcs. But if the codec is disabled for simulcast
2473 // or a screencast (and not in simulcast screenshare experiment), only
2474 // configure a single stream.
2475 encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
2476 if (IsCodecDisabledForSimulcast(codec.name, call_->trials())) {
2477 encoder_config.number_of_streams = 1;
2478 }
2479
2480 // parameters_.max_bitrate comes from the max bitrate set at the SDP
2481 // (m-section) level with the attribute "b=AS." Note that we override this
2482 // value below if the RtpParameters max bitrate set with
2483 // RtpSender::SetParameters has a lower value.
2484 int stream_max_bitrate = parameters_.max_bitrate_bps;
2485 // When simulcast is enabled (when there are multiple encodings),
2486 // encodings[i].max_bitrate_bps will be enforced by
2487 // encoder_config.simulcast_layers[i].max_bitrate_bps. Otherwise, it's
2488 // enforced by stream_max_bitrate, taking the minimum of the two maximums
2489 // (one coming from SDP, the other coming from RtpParameters).
2490 if (rtp_parameters_.encodings[0].max_bitrate_bps &&
2491 rtp_parameters_.encodings.size() == 1) {
2492 stream_max_bitrate =
2493 MinPositive(*(rtp_parameters_.encodings[0].max_bitrate_bps),
2494 parameters_.max_bitrate_bps);
2495 }
2496
2497 // The codec max bitrate comes from the "x-google-max-bitrate" parameter
2498 // attribute set in the SDP for a specific codec. As done in
2499 // WebRtcVideoChannel::SetSendParameters, this value does not override the
2500 // stream max_bitrate set above.
2501 int codec_max_bitrate_kbps;
2502 if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps) &&
2503 stream_max_bitrate == -1) {
2504 stream_max_bitrate = codec_max_bitrate_kbps * 1000;
2505 }
2506 encoder_config.max_bitrate_bps = stream_max_bitrate;
2507
2508 // The encoder config's default bitrate priority is set to 1.0,
2509 // unless it is set through the sender's encoding parameters.
2510 // The bitrate priority, which is used in the bitrate allocation, is done
2511 // on a per sender basis, so we use the first encoding's value.
2512 encoder_config.bitrate_priority =
2513 rtp_parameters_.encodings[0].bitrate_priority;
2514
2515 // Application-controlled state is held in the encoder_config's
2516 // simulcast_layers. Currently this is used to control which simulcast layers
2517 // are active and for configuring the min/max bitrate and max framerate.
2518 // The encoder_config's simulcast_layers is also used for non-simulcast (when
2519 // there is a single layer).
2520 RTC_DCHECK_GE(rtp_parameters_.encodings.size(),
2521 encoder_config.number_of_streams);
2522 RTC_DCHECK_GT(encoder_config.number_of_streams, 0);
2523
2524 // Copy all provided constraints.
2525 encoder_config.simulcast_layers.resize(rtp_parameters_.encodings.size());
2526 for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
2527 encoder_config.simulcast_layers[i].active =
2528 rtp_parameters_.encodings[i].active;
2529 encoder_config.simulcast_layers[i].scalability_mode =
2530 webrtc::ScalabilityModeFromString(
2531 rtp_parameters_.encodings[i].scalability_mode.value_or(""));
2532 if (rtp_parameters_.encodings[i].min_bitrate_bps) {
2533 encoder_config.simulcast_layers[i].min_bitrate_bps =
2534 *rtp_parameters_.encodings[i].min_bitrate_bps;
2535 }
2536 if (rtp_parameters_.encodings[i].max_bitrate_bps) {
2537 encoder_config.simulcast_layers[i].max_bitrate_bps =
2538 *rtp_parameters_.encodings[i].max_bitrate_bps;
2539 }
2540 if (rtp_parameters_.encodings[i].max_framerate) {
2541 encoder_config.simulcast_layers[i].max_framerate =
2542 *rtp_parameters_.encodings[i].max_framerate;
2543 }
2544 if (rtp_parameters_.encodings[i].scale_resolution_down_by) {
2545 encoder_config.simulcast_layers[i].scale_resolution_down_by =
2546 *rtp_parameters_.encodings[i].scale_resolution_down_by;
2547 }
2548 if (rtp_parameters_.encodings[i].num_temporal_layers) {
2549 encoder_config.simulcast_layers[i].num_temporal_layers =
2550 *rtp_parameters_.encodings[i].num_temporal_layers;
2551 }
2552 encoder_config.simulcast_layers[i].requested_resolution =
2553 rtp_parameters_.encodings[i].requested_resolution;
2554 }
2555
2556 encoder_config.legacy_conference_mode = parameters_.conference_mode;
2557
2558 encoder_config.is_quality_scaling_allowed =
2559 !disable_automatic_resize_ && !is_screencast &&
2560 (parameters_.config.rtp.ssrcs.size() == 1 ||
2561 NumActiveStreams(rtp_parameters_) == 1);
2562
2563 // Ensure frame dropping is always enabled.
2564 encoder_config.frame_drop_enabled = true;
2565
2566 int max_qp = kDefaultQpMax;
2567 codec.GetParam(kCodecParamMaxQuantization, &max_qp);
2568 encoder_config.max_qp = max_qp;
2569
2570 return encoder_config;
2571 }
2572
ReconfigureEncoder(webrtc::SetParametersCallback callback)2573 void WebRtcVideoChannel::WebRtcVideoSendStream::ReconfigureEncoder(
2574 webrtc::SetParametersCallback callback) {
2575 RTC_DCHECK_RUN_ON(&thread_checker_);
2576 if (!stream_) {
2577 // The webrtc::VideoSendStream `stream_` has not yet been created but other
2578 // parameters has changed.
2579 webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
2580 return;
2581 }
2582
2583 RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
2584
2585 RTC_CHECK(parameters_.codec_settings);
2586 VideoCodecSettings codec_settings = *parameters_.codec_settings;
2587
2588 webrtc::VideoEncoderConfig encoder_config =
2589 CreateVideoEncoderConfig(codec_settings.codec);
2590
2591 encoder_config.encoder_specific_settings =
2592 ConfigureVideoEncoderSettings(codec_settings.codec);
2593
2594 stream_->ReconfigureVideoEncoder(encoder_config.Copy(), std::move(callback));
2595
2596 encoder_config.encoder_specific_settings = NULL;
2597
2598 parameters_.encoder_config = std::move(encoder_config);
2599 }
2600
SetSend(bool send)2601 void WebRtcVideoChannel::WebRtcVideoSendStream::SetSend(bool send) {
2602 RTC_DCHECK_RUN_ON(&thread_checker_);
2603 sending_ = send;
2604 UpdateSendState();
2605 }
2606
2607 std::vector<VideoSenderInfo>
GetPerLayerVideoSenderInfos(bool log_stats)2608 WebRtcVideoChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos(
2609 bool log_stats) {
2610 RTC_DCHECK_RUN_ON(&thread_checker_);
2611 VideoSenderInfo common_info;
2612 if (parameters_.codec_settings) {
2613 common_info.codec_name = parameters_.codec_settings->codec.name;
2614 common_info.codec_payload_type = parameters_.codec_settings->codec.id;
2615 }
2616 std::vector<VideoSenderInfo> infos;
2617 webrtc::VideoSendStream::Stats stats;
2618 if (stream_ == nullptr) {
2619 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2620 common_info.add_ssrc(ssrc);
2621 }
2622 infos.push_back(common_info);
2623 return infos;
2624 } else {
2625 stats = stream_->GetStats();
2626 if (log_stats)
2627 RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
2628
2629 // Metrics that are in common for all substreams.
2630 common_info.adapt_changes = stats.number_of_cpu_adapt_changes;
2631 common_info.adapt_reason =
2632 stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE;
2633 common_info.has_entered_low_resolution = stats.has_entered_low_resolution;
2634
2635 // Get bandwidth limitation info from stream_->GetStats().
2636 // Input resolution (output from video_adapter) can be further scaled down
2637 // or higher video layer(s) can be dropped due to bitrate constraints.
2638 // Note, adapt_changes only include changes from the video_adapter.
2639 if (stats.bw_limited_resolution)
2640 common_info.adapt_reason |= ADAPTREASON_BANDWIDTH;
2641
2642 common_info.quality_limitation_reason = stats.quality_limitation_reason;
2643 common_info.quality_limitation_durations_ms =
2644 stats.quality_limitation_durations_ms;
2645 common_info.quality_limitation_resolution_changes =
2646 stats.quality_limitation_resolution_changes;
2647 common_info.encoder_implementation_name = stats.encoder_implementation_name;
2648 common_info.target_bitrate = stats.target_media_bitrate_bps;
2649 common_info.ssrc_groups = ssrc_groups_;
2650 common_info.frames = stats.frames;
2651 common_info.framerate_input = stats.input_frame_rate;
2652 common_info.avg_encode_ms = stats.avg_encode_time_ms;
2653 common_info.encode_usage_percent = stats.encode_usage_percent;
2654 common_info.nominal_bitrate = stats.media_bitrate_bps;
2655 common_info.content_type = stats.content_type;
2656 common_info.aggregated_framerate_sent = stats.encode_frame_rate;
2657 common_info.aggregated_huge_frames_sent = stats.huge_frames_sent;
2658 common_info.power_efficient_encoder = stats.power_efficient_encoder;
2659
2660 // The normal case is that substreams are present, handled below. But if
2661 // substreams are missing (can happen before negotiated/connected where we
2662 // have no stats yet) a single outbound-rtp is created representing any and
2663 // all layers.
2664 if (stats.substreams.empty()) {
2665 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2666 common_info.add_ssrc(ssrc);
2667 }
2668 common_info.active =
2669 IsActiveFromEncodings(absl::nullopt, rtp_parameters_.encodings);
2670 common_info.framerate_sent = stats.encode_frame_rate;
2671 common_info.frames_encoded = stats.frames_encoded;
2672 common_info.total_encode_time_ms = stats.total_encode_time_ms;
2673 common_info.total_encoded_bytes_target = stats.total_encoded_bytes_target;
2674 common_info.frames_sent = stats.frames_encoded;
2675 common_info.huge_frames_sent = stats.huge_frames_sent;
2676 infos.push_back(common_info);
2677 return infos;
2678 }
2679 }
2680 // Merge `stats.substreams`, which may contain additional SSRCs for RTX or
2681 // Flexfec, with media SSRCs. This results in a set of substreams that match
2682 // with the outbound-rtp stats objects.
2683 auto outbound_rtp_substreams =
2684 MergeInfoAboutOutboundRtpSubstreams(stats.substreams);
2685 // If SVC is used, one stream is configured but multiple encodings exist. This
2686 // is not spec-compliant, but it is how we've implemented SVC so this affects
2687 // how the RTP stream's "active" value is determined.
2688 bool is_svc = (parameters_.encoder_config.number_of_streams == 1 &&
2689 rtp_parameters_.encodings.size() > 1);
2690 for (const auto& pair : outbound_rtp_substreams) {
2691 auto info = common_info;
2692 uint32_t ssrc = pair.first;
2693 info.add_ssrc(ssrc);
2694 info.rid = parameters_.config.rtp.GetRidForSsrc(ssrc);
2695 info.active = IsActiveFromEncodings(
2696 !is_svc ? absl::optional<uint32_t>(ssrc) : absl::nullopt,
2697 rtp_parameters_.encodings);
2698 auto stream_stats = pair.second;
2699 RTC_DCHECK_EQ(stream_stats.type,
2700 webrtc::VideoSendStream::StreamStats::StreamType::kMedia);
2701 info.payload_bytes_sent = stream_stats.rtp_stats.transmitted.payload_bytes;
2702 info.header_and_padding_bytes_sent =
2703 stream_stats.rtp_stats.transmitted.header_bytes +
2704 stream_stats.rtp_stats.transmitted.padding_bytes;
2705 info.packets_sent = stream_stats.rtp_stats.transmitted.packets;
2706 info.total_packet_send_delay +=
2707 stream_stats.rtp_stats.transmitted.total_packet_delay;
2708 info.send_frame_width = stream_stats.width;
2709 info.send_frame_height = stream_stats.height;
2710 info.key_frames_encoded = stream_stats.frame_counts.key_frames;
2711 info.framerate_sent = stream_stats.encode_frame_rate;
2712 info.frames_encoded = stream_stats.frames_encoded;
2713 info.frames_sent = stream_stats.frames_encoded;
2714 info.retransmitted_bytes_sent =
2715 stream_stats.rtp_stats.retransmitted.payload_bytes;
2716 info.retransmitted_packets_sent =
2717 stream_stats.rtp_stats.retransmitted.packets;
2718 info.firs_rcvd = stream_stats.rtcp_packet_type_counts.fir_packets;
2719 info.nacks_rcvd = stream_stats.rtcp_packet_type_counts.nack_packets;
2720 info.plis_rcvd = stream_stats.rtcp_packet_type_counts.pli_packets;
2721 if (stream_stats.report_block_data.has_value()) {
2722 info.packets_lost =
2723 stream_stats.report_block_data->report_block().packets_lost;
2724 info.fraction_lost =
2725 static_cast<float>(
2726 stream_stats.report_block_data->report_block().fraction_lost) /
2727 (1 << 8);
2728 info.report_block_datas.push_back(*stream_stats.report_block_data);
2729 }
2730 info.qp_sum = stream_stats.qp_sum;
2731 info.total_encode_time_ms = stream_stats.total_encode_time_ms;
2732 info.total_encoded_bytes_target = stream_stats.total_encoded_bytes_target;
2733 info.huge_frames_sent = stream_stats.huge_frames_sent;
2734 infos.push_back(info);
2735 }
2736 return infos;
2737 }
2738
2739 VideoSenderInfo
GetAggregatedVideoSenderInfo(const std::vector<VideoSenderInfo> & infos) const2740 WebRtcVideoChannel::WebRtcVideoSendStream::GetAggregatedVideoSenderInfo(
2741 const std::vector<VideoSenderInfo>& infos) const {
2742 RTC_DCHECK_RUN_ON(&thread_checker_);
2743 RTC_CHECK(!infos.empty());
2744 if (infos.size() == 1) {
2745 return infos[0];
2746 }
2747 VideoSenderInfo info = infos[0];
2748 info.local_stats.clear();
2749 for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2750 info.add_ssrc(ssrc);
2751 }
2752 info.framerate_sent = info.aggregated_framerate_sent;
2753 info.huge_frames_sent = info.aggregated_huge_frames_sent;
2754
2755 for (size_t i = 1; i < infos.size(); i++) {
2756 info.key_frames_encoded += infos[i].key_frames_encoded;
2757 info.payload_bytes_sent += infos[i].payload_bytes_sent;
2758 info.header_and_padding_bytes_sent +=
2759 infos[i].header_and_padding_bytes_sent;
2760 info.packets_sent += infos[i].packets_sent;
2761 info.total_packet_send_delay += infos[i].total_packet_send_delay;
2762 info.retransmitted_bytes_sent += infos[i].retransmitted_bytes_sent;
2763 info.retransmitted_packets_sent += infos[i].retransmitted_packets_sent;
2764 info.packets_lost += infos[i].packets_lost;
2765 if (infos[i].send_frame_width > info.send_frame_width)
2766 info.send_frame_width = infos[i].send_frame_width;
2767 if (infos[i].send_frame_height > info.send_frame_height)
2768 info.send_frame_height = infos[i].send_frame_height;
2769 info.firs_rcvd += infos[i].firs_rcvd;
2770 info.nacks_rcvd += infos[i].nacks_rcvd;
2771 info.plis_rcvd += infos[i].plis_rcvd;
2772 if (infos[i].report_block_datas.size())
2773 info.report_block_datas.push_back(infos[i].report_block_datas[0]);
2774 if (infos[i].qp_sum) {
2775 if (!info.qp_sum) {
2776 info.qp_sum = 0;
2777 }
2778 info.qp_sum = *info.qp_sum + *infos[i].qp_sum;
2779 }
2780 info.frames_encoded += infos[i].frames_encoded;
2781 info.frames_sent += infos[i].frames_sent;
2782 info.total_encode_time_ms += infos[i].total_encode_time_ms;
2783 info.total_encoded_bytes_target += infos[i].total_encoded_bytes_target;
2784 }
2785 return info;
2786 }
2787
FillBitrateInfo(BandwidthEstimationInfo * bwe_info)2788 void WebRtcVideoChannel::WebRtcVideoSendStream::FillBitrateInfo(
2789 BandwidthEstimationInfo* bwe_info) {
2790 RTC_DCHECK_RUN_ON(&thread_checker_);
2791 if (stream_ == NULL) {
2792 return;
2793 }
2794 webrtc::VideoSendStream::Stats stats = stream_->GetStats();
2795 for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it =
2796 stats.substreams.begin();
2797 it != stats.substreams.end(); ++it) {
2798 bwe_info->transmit_bitrate += it->second.total_bitrate_bps;
2799 bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps;
2800 }
2801 bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps;
2802 bwe_info->actual_enc_bitrate += stats.media_bitrate_bps;
2803 }
2804
2805 void WebRtcVideoChannel::WebRtcVideoSendStream::
SetEncoderToPacketizerFrameTransformer(rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)2806 SetEncoderToPacketizerFrameTransformer(
2807 rtc::scoped_refptr<webrtc::FrameTransformerInterface>
2808 frame_transformer) {
2809 RTC_DCHECK_RUN_ON(&thread_checker_);
2810 parameters_.config.frame_transformer = std::move(frame_transformer);
2811 if (stream_)
2812 RecreateWebRtcStream();
2813 }
2814
RecreateWebRtcStream()2815 void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
2816 RTC_DCHECK_RUN_ON(&thread_checker_);
2817 if (stream_ != NULL) {
2818 call_->DestroyVideoSendStream(stream_);
2819 }
2820
2821 RTC_CHECK(parameters_.codec_settings);
2822 RTC_DCHECK_EQ((parameters_.encoder_config.content_type ==
2823 webrtc::VideoEncoderConfig::ContentType::kScreen),
2824 parameters_.options.is_screencast.value_or(false))
2825 << "encoder content type inconsistent with screencast option";
2826 parameters_.encoder_config.encoder_specific_settings =
2827 ConfigureVideoEncoderSettings(parameters_.codec_settings->codec);
2828
2829 webrtc::VideoSendStream::Config config = parameters_.config.Copy();
2830 if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) {
2831 RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
2832 "payload type the set codec. Ignoring RTX.";
2833 config.rtp.rtx.ssrcs.clear();
2834 }
2835 if (parameters_.encoder_config.number_of_streams == 1) {
2836 // SVC is used instead of simulcast. Remove unnecessary SSRCs.
2837 if (config.rtp.ssrcs.size() > 1) {
2838 config.rtp.ssrcs.resize(1);
2839 if (config.rtp.rtx.ssrcs.size() > 1) {
2840 config.rtp.rtx.ssrcs.resize(1);
2841 }
2842 }
2843 }
2844 stream_ = call_->CreateVideoSendStream(std::move(config),
2845 parameters_.encoder_config.Copy());
2846
2847 parameters_.encoder_config.encoder_specific_settings = NULL;
2848
2849 // Calls stream_->StartPerRtpStream() to start the VideoSendStream
2850 // if necessary conditions are met.
2851 UpdateSendState();
2852
2853 // Attach the source after starting the send stream to prevent frames from
2854 // being injected into a not-yet initializated video stream encoder.
2855 if (source_) {
2856 stream_->SetSource(source_, GetDegradationPreference());
2857 }
2858 }
2859
GenerateKeyFrame(const std::vector<std::string> & rids)2860 void WebRtcVideoChannel::WebRtcVideoSendStream::GenerateKeyFrame(
2861 const std::vector<std::string>& rids) {
2862 RTC_DCHECK_RUN_ON(&thread_checker_);
2863 if (stream_ != NULL) {
2864 stream_->GenerateKeyFrame(rids);
2865 } else {
2866 RTC_LOG(LS_WARNING)
2867 << "Absent send stream; ignoring request to generate keyframe.";
2868 }
2869 }
2870
WebRtcVideoReceiveStream(WebRtcVideoChannel * channel,webrtc::Call * call,const StreamParams & sp,webrtc::VideoReceiveStreamInterface::Config config,bool default_stream,const std::vector<VideoCodecSettings> & recv_codecs,const webrtc::FlexfecReceiveStream::Config & flexfec_config)2871 WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
2872 WebRtcVideoChannel* channel,
2873 webrtc::Call* call,
2874 const StreamParams& sp,
2875 webrtc::VideoReceiveStreamInterface::Config config,
2876 bool default_stream,
2877 const std::vector<VideoCodecSettings>& recv_codecs,
2878 const webrtc::FlexfecReceiveStream::Config& flexfec_config)
2879 : channel_(channel),
2880 call_(call),
2881 stream_params_(sp),
2882 stream_(NULL),
2883 default_stream_(default_stream),
2884 config_(std::move(config)),
2885 flexfec_config_(flexfec_config),
2886 flexfec_stream_(nullptr),
2887 sink_(NULL),
2888 first_frame_timestamp_(-1),
2889 estimated_remote_start_ntp_time_ms_(0) {
2890 RTC_DCHECK(config_.decoder_factory);
2891 RTC_DCHECK(config_.decoders.empty())
2892 << "Decoder info is supplied via `recv_codecs`";
2893
2894 ExtractCodecInformation(recv_codecs, config_.rtp.rtx_associated_payload_types,
2895 config_.rtp.raw_payload_types, config_.decoders);
2896 const VideoCodecSettings& codec = recv_codecs.front();
2897 config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type;
2898 config_.rtp.red_payload_type = codec.ulpfec.red_payload_type;
2899 config_.rtp.lntf.enabled = HasLntf(codec.codec);
2900 config_.rtp.nack.rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0;
2901 if (codec.rtx_time != -1 && config_.rtp.nack.rtp_history_ms != 0) {
2902 config_.rtp.nack.rtp_history_ms = codec.rtx_time;
2903 }
2904
2905 config_.rtp.rtcp_xr.receiver_reference_time_report = HasRrtr(codec.codec);
2906
2907 if (codec.ulpfec.red_rtx_payload_type != -1) {
2908 config_.rtp
2909 .rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] =
2910 codec.ulpfec.red_payload_type;
2911 }
2912
2913 config_.renderer = this;
2914 flexfec_config_.payload_type = flexfec_config.payload_type;
2915
2916 CreateReceiveStream();
2917 StartReceiveStream();
2918 }
2919
~WebRtcVideoReceiveStream()2920 WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
2921 call_->DestroyVideoReceiveStream(stream_);
2922 if (flexfec_stream_)
2923 call_->DestroyFlexfecReceiveStream(flexfec_stream_);
2924 }
2925
2926 webrtc::VideoReceiveStreamInterface&
stream()2927 WebRtcVideoChannel::WebRtcVideoReceiveStream::stream() {
2928 RTC_DCHECK(stream_);
2929 return *stream_;
2930 }
2931
2932 webrtc::FlexfecReceiveStream*
flexfec_stream()2933 WebRtcVideoChannel::WebRtcVideoReceiveStream::flexfec_stream() {
2934 return flexfec_stream_;
2935 }
2936
2937 const std::vector<uint32_t>&
GetSsrcs() const2938 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSsrcs() const {
2939 return stream_params_.ssrcs;
2940 }
2941
2942 std::vector<webrtc::RtpSource>
GetSources()2943 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSources() {
2944 RTC_DCHECK(stream_);
2945 return stream_->GetSources();
2946 }
2947
2948 webrtc::RtpParameters
GetRtpParameters() const2949 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetRtpParameters() const {
2950 webrtc::RtpParameters rtp_parameters;
2951
2952 std::vector<uint32_t> primary_ssrcs;
2953 stream_params_.GetPrimarySsrcs(&primary_ssrcs);
2954 for (uint32_t ssrc : primary_ssrcs) {
2955 rtp_parameters.encodings.emplace_back();
2956 rtp_parameters.encodings.back().ssrc = ssrc;
2957 }
2958
2959 rtp_parameters.header_extensions = config_.rtp.extensions;
2960 rtp_parameters.rtcp.reduced_size =
2961 config_.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
2962
2963 return rtp_parameters;
2964 }
2965
ReconfigureCodecs(const std::vector<VideoCodecSettings> & recv_codecs)2966 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::ReconfigureCodecs(
2967 const std::vector<VideoCodecSettings>& recv_codecs) {
2968 RTC_DCHECK(stream_);
2969 RTC_DCHECK(!recv_codecs.empty());
2970
2971 std::map<int, int> rtx_associated_payload_types;
2972 std::set<int> raw_payload_types;
2973 std::vector<webrtc::VideoReceiveStreamInterface::Decoder> decoders;
2974 ExtractCodecInformation(recv_codecs, rtx_associated_payload_types,
2975 raw_payload_types, decoders);
2976
2977 const auto& codec = recv_codecs.front();
2978
2979 if (config_.rtp.red_payload_type != codec.ulpfec.red_payload_type ||
2980 config_.rtp.ulpfec_payload_type != codec.ulpfec.ulpfec_payload_type) {
2981 config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type;
2982 config_.rtp.red_payload_type = codec.ulpfec.red_payload_type;
2983 stream_->SetProtectionPayloadTypes(config_.rtp.red_payload_type,
2984 config_.rtp.ulpfec_payload_type);
2985 }
2986
2987 const bool has_lntf = HasLntf(codec.codec);
2988 if (config_.rtp.lntf.enabled != has_lntf) {
2989 config_.rtp.lntf.enabled = has_lntf;
2990 stream_->SetLossNotificationEnabled(has_lntf);
2991 }
2992
2993 int new_history_ms = config_.rtp.nack.rtp_history_ms;
2994 const int rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0;
2995 if (rtp_history_ms != config_.rtp.nack.rtp_history_ms) {
2996 new_history_ms = rtp_history_ms;
2997 }
2998
2999 // The rtx-time parameter can be used to override the hardcoded default for
3000 // the NACK buffer length.
3001 if (codec.rtx_time != -1 && new_history_ms != 0) {
3002 new_history_ms = codec.rtx_time;
3003 }
3004
3005 if (config_.rtp.nack.rtp_history_ms != new_history_ms) {
3006 config_.rtp.nack.rtp_history_ms = new_history_ms;
3007 stream_->SetNackHistory(webrtc::TimeDelta::Millis(new_history_ms));
3008 }
3009
3010 const bool has_rtr = HasRrtr(codec.codec);
3011 if (has_rtr != config_.rtp.rtcp_xr.receiver_reference_time_report) {
3012 config_.rtp.rtcp_xr.receiver_reference_time_report = has_rtr;
3013 stream_->SetRtcpXr(config_.rtp.rtcp_xr);
3014 }
3015
3016 if (codec.ulpfec.red_rtx_payload_type != -1) {
3017 rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] =
3018 codec.ulpfec.red_payload_type;
3019 }
3020
3021 if (config_.rtp.rtx_associated_payload_types !=
3022 rtx_associated_payload_types) {
3023 stream_->SetAssociatedPayloadTypes(rtx_associated_payload_types);
3024 rtx_associated_payload_types.swap(config_.rtp.rtx_associated_payload_types);
3025 }
3026
3027 bool recreate_needed = false;
3028
3029 if (raw_payload_types != config_.rtp.raw_payload_types) {
3030 raw_payload_types.swap(config_.rtp.raw_payload_types);
3031 recreate_needed = true;
3032 }
3033
3034 if (decoders != config_.decoders) {
3035 decoders.swap(config_.decoders);
3036 recreate_needed = true;
3037 }
3038
3039 return recreate_needed;
3040 }
3041
SetFeedbackParameters(bool lntf_enabled,bool nack_enabled,bool transport_cc_enabled,webrtc::RtcpMode rtcp_mode,int rtx_time)3042 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFeedbackParameters(
3043 bool lntf_enabled,
3044 bool nack_enabled,
3045 bool transport_cc_enabled,
3046 webrtc::RtcpMode rtcp_mode,
3047 int rtx_time) {
3048 RTC_DCHECK(stream_);
3049
3050 if (config_.rtp.rtcp_mode != rtcp_mode) {
3051 config_.rtp.rtcp_mode = rtcp_mode;
3052 stream_->SetRtcpMode(rtcp_mode);
3053
3054 flexfec_config_.rtcp_mode = rtcp_mode;
3055 if (flexfec_stream_) {
3056 flexfec_stream_->SetRtcpMode(rtcp_mode);
3057 }
3058 }
3059
3060 if (config_.rtp.transport_cc != transport_cc_enabled) {
3061 config_.rtp.transport_cc = transport_cc_enabled;
3062 stream_->SetTransportCc(transport_cc_enabled);
3063 // TODO(brandtr): We should be spec-compliant and set `transport_cc` here
3064 // based on the rtcp-fb for the FlexFEC codec, not the media codec.
3065 flexfec_config_.rtp.transport_cc = transport_cc_enabled;
3066 if (flexfec_stream_) {
3067 flexfec_stream_->SetTransportCc(transport_cc_enabled);
3068 }
3069 }
3070
3071 config_.rtp.lntf.enabled = lntf_enabled;
3072 stream_->SetLossNotificationEnabled(lntf_enabled);
3073
3074 int nack_history_ms =
3075 nack_enabled ? rtx_time != -1 ? rtx_time : kNackHistoryMs : 0;
3076 config_.rtp.nack.rtp_history_ms = nack_history_ms;
3077 stream_->SetNackHistory(webrtc::TimeDelta::Millis(nack_history_ms));
3078 }
3079
SetFlexFecPayload(int payload_type)3080 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFlexFecPayload(
3081 int payload_type) {
3082 // TODO(bugs.webrtc.org/11993, tommi): See if it is better to always have a
3083 // flexfec stream object around and instead of recreating the video stream,
3084 // reconfigure the flexfec object from within the rtp callback (soon to be on
3085 // the network thread).
3086 if (flexfec_stream_) {
3087 if (flexfec_stream_->payload_type() == payload_type) {
3088 RTC_DCHECK_EQ(flexfec_config_.payload_type, payload_type);
3089 return;
3090 }
3091
3092 flexfec_config_.payload_type = payload_type;
3093 flexfec_stream_->SetPayloadType(payload_type);
3094
3095 if (payload_type == -1) {
3096 stream_->SetFlexFecProtection(nullptr);
3097 call_->DestroyFlexfecReceiveStream(flexfec_stream_);
3098 flexfec_stream_ = nullptr;
3099 }
3100 } else if (payload_type != -1) {
3101 flexfec_config_.payload_type = payload_type;
3102 if (flexfec_config_.IsCompleteAndEnabled()) {
3103 flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
3104 stream_->SetFlexFecProtection(flexfec_stream_);
3105 }
3106 } else {
3107 // Noop. No flexfec stream exists and "new" payload_type == -1.
3108 RTC_DCHECK(!flexfec_config_.IsCompleteAndEnabled());
3109 flexfec_config_.payload_type = payload_type;
3110 }
3111 }
3112
SetRecvParameters(const ChangedRecvParameters & params)3113 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
3114 const ChangedRecvParameters& params) {
3115 RTC_DCHECK(stream_);
3116 bool video_needs_recreation = false;
3117 if (params.codec_settings) {
3118 video_needs_recreation = ReconfigureCodecs(*params.codec_settings);
3119 }
3120
3121 if (params.rtp_header_extensions) {
3122 if (config_.rtp.extensions != *params.rtp_header_extensions) {
3123 config_.rtp.extensions = *params.rtp_header_extensions;
3124 stream_->SetRtpExtensions(config_.rtp.extensions);
3125 }
3126
3127 if (flexfec_config_.rtp.extensions != *params.rtp_header_extensions) {
3128 flexfec_config_.rtp.extensions = *params.rtp_header_extensions;
3129 if (flexfec_stream_) {
3130 flexfec_stream_->SetRtpExtensions(flexfec_config_.rtp.extensions);
3131 }
3132 }
3133 }
3134
3135 if (params.flexfec_payload_type)
3136 SetFlexFecPayload(*params.flexfec_payload_type);
3137
3138 if (video_needs_recreation) {
3139 RecreateReceiveStream();
3140 } else {
3141 RTC_DLOG_F(LS_INFO) << "No receive stream recreate needed.";
3142 }
3143 }
3144
RecreateReceiveStream()3145 void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() {
3146 RTC_DCHECK(stream_);
3147 absl::optional<int> base_minimum_playout_delay_ms;
3148 absl::optional<webrtc::VideoReceiveStreamInterface::RecordingState>
3149 recording_state;
3150 if (stream_) {
3151 base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs();
3152 recording_state = stream_->SetAndGetRecordingState(
3153 webrtc::VideoReceiveStreamInterface::RecordingState(),
3154 /*generate_key_frame=*/false);
3155 call_->DestroyVideoReceiveStream(stream_);
3156 stream_ = nullptr;
3157 }
3158
3159 if (flexfec_stream_) {
3160 call_->DestroyFlexfecReceiveStream(flexfec_stream_);
3161 flexfec_stream_ = nullptr;
3162 }
3163
3164 CreateReceiveStream();
3165
3166 if (base_minimum_playout_delay_ms) {
3167 stream_->SetBaseMinimumPlayoutDelayMs(
3168 base_minimum_playout_delay_ms.value());
3169 }
3170 if (recording_state) {
3171 stream_->SetAndGetRecordingState(std::move(*recording_state),
3172 /*generate_key_frame=*/false);
3173 }
3174
3175 StartReceiveStream();
3176 }
3177
CreateReceiveStream()3178 void WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateReceiveStream() {
3179 RTC_DCHECK(!stream_);
3180 RTC_DCHECK(!flexfec_stream_);
3181 if (flexfec_config_.IsCompleteAndEnabled()) {
3182 flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
3183 }
3184
3185 webrtc::VideoReceiveStreamInterface::Config config = config_.Copy();
3186 config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr);
3187 config.rtp.packet_sink_ = flexfec_stream_;
3188 stream_ = call_->CreateVideoReceiveStream(std::move(config));
3189 }
3190
StartReceiveStream()3191 void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartReceiveStream() {
3192 stream_->Start();
3193 if (IsEnabled(call_->trials(), "WebRTC-Video-BufferPacketsWithUnknownSsrc")) {
3194 channel_->BackfillBufferedPackets(stream_params_.ssrcs);
3195 }
3196 }
3197
OnFrame(const webrtc::VideoFrame & frame)3198 void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame(
3199 const webrtc::VideoFrame& frame) {
3200 webrtc::MutexLock lock(&sink_lock_);
3201
3202 int64_t time_now_ms = rtc::TimeMillis();
3203 if (first_frame_timestamp_ < 0)
3204 first_frame_timestamp_ = time_now_ms;
3205 int64_t elapsed_time_ms = time_now_ms - first_frame_timestamp_;
3206 if (frame.ntp_time_ms() > 0)
3207 estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms;
3208
3209 if (sink_ == NULL) {
3210 RTC_LOG(LS_WARNING)
3211 << "VideoReceiveStreamInterface not connected to a VideoSink.";
3212 return;
3213 }
3214
3215 sink_->OnFrame(frame);
3216 }
3217
IsDefaultStream() const3218 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::IsDefaultStream() const {
3219 return default_stream_;
3220 }
3221
SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)3222 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFrameDecryptor(
3223 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
3224 config_.frame_decryptor = frame_decryptor;
3225 if (stream_) {
3226 RTC_LOG(LS_INFO)
3227 << "Setting FrameDecryptor (recv) because of SetFrameDecryptor, "
3228 "remote_ssrc="
3229 << config_.rtp.remote_ssrc;
3230 stream_->SetFrameDecryptor(frame_decryptor);
3231 }
3232 }
3233
SetBaseMinimumPlayoutDelayMs(int delay_ms)3234 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::SetBaseMinimumPlayoutDelayMs(
3235 int delay_ms) {
3236 return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false;
3237 }
3238
GetBaseMinimumPlayoutDelayMs() const3239 int WebRtcVideoChannel::WebRtcVideoReceiveStream::GetBaseMinimumPlayoutDelayMs()
3240 const {
3241 return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0;
3242 }
3243
SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)3244 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetSink(
3245 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
3246 webrtc::MutexLock lock(&sink_lock_);
3247 sink_ = sink;
3248 }
3249
3250 VideoReceiverInfo
GetVideoReceiverInfo(bool log_stats)3251 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo(
3252 bool log_stats) {
3253 VideoReceiverInfo info;
3254 info.ssrc_groups = stream_params_.ssrc_groups;
3255 info.add_ssrc(config_.rtp.remote_ssrc);
3256 webrtc::VideoReceiveStreamInterface::Stats stats = stream_->GetStats();
3257 info.decoder_implementation_name = stats.decoder_implementation_name;
3258 info.power_efficient_decoder = stats.power_efficient_decoder;
3259 if (stats.current_payload_type != -1) {
3260 info.codec_payload_type = stats.current_payload_type;
3261 auto decoder_it = absl::c_find_if(config_.decoders, [&](const auto& d) {
3262 return d.payload_type == stats.current_payload_type;
3263 });
3264 if (decoder_it != config_.decoders.end())
3265 info.codec_name = decoder_it->video_format.name;
3266 }
3267 info.payload_bytes_rcvd = stats.rtp_stats.packet_counter.payload_bytes;
3268 info.header_and_padding_bytes_rcvd =
3269 stats.rtp_stats.packet_counter.header_bytes +
3270 stats.rtp_stats.packet_counter.padding_bytes;
3271 info.packets_rcvd = stats.rtp_stats.packet_counter.packets;
3272 info.packets_lost = stats.rtp_stats.packets_lost;
3273 info.jitter_ms = stats.rtp_stats.jitter / (kVideoCodecClockrate / 1000);
3274
3275 info.framerate_rcvd = stats.network_frame_rate;
3276 info.framerate_decoded = stats.decode_frame_rate;
3277 info.framerate_output = stats.render_frame_rate;
3278 info.frame_width = stats.width;
3279 info.frame_height = stats.height;
3280
3281 {
3282 webrtc::MutexLock frame_cs(&sink_lock_);
3283 info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_;
3284 }
3285
3286 info.decode_ms = stats.decode_ms;
3287 info.max_decode_ms = stats.max_decode_ms;
3288 info.current_delay_ms = stats.current_delay_ms;
3289 info.target_delay_ms = stats.target_delay_ms;
3290 info.jitter_buffer_ms = stats.jitter_buffer_ms;
3291 info.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds;
3292 info.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count;
3293 info.min_playout_delay_ms = stats.min_playout_delay_ms;
3294 info.render_delay_ms = stats.render_delay_ms;
3295 info.frames_received =
3296 stats.frame_counts.key_frames + stats.frame_counts.delta_frames;
3297 info.frames_dropped = stats.frames_dropped;
3298 info.frames_decoded = stats.frames_decoded;
3299 info.key_frames_decoded = stats.frame_counts.key_frames;
3300 info.frames_rendered = stats.frames_rendered;
3301 info.qp_sum = stats.qp_sum;
3302 info.total_decode_time = stats.total_decode_time;
3303 info.total_processing_delay = stats.total_processing_delay;
3304 info.total_assembly_time = stats.total_assembly_time;
3305 info.frames_assembled_from_multiple_packets =
3306 stats.frames_assembled_from_multiple_packets;
3307 info.last_packet_received_timestamp_ms =
3308 stats.rtp_stats.last_packet_received_timestamp_ms;
3309 info.estimated_playout_ntp_timestamp_ms =
3310 stats.estimated_playout_ntp_timestamp_ms;
3311 info.first_frame_received_to_decoded_ms =
3312 stats.first_frame_received_to_decoded_ms;
3313 info.total_inter_frame_delay = stats.total_inter_frame_delay;
3314 info.total_squared_inter_frame_delay = stats.total_squared_inter_frame_delay;
3315 info.interframe_delay_max_ms = stats.interframe_delay_max_ms;
3316 info.freeze_count = stats.freeze_count;
3317 info.pause_count = stats.pause_count;
3318 info.total_freezes_duration_ms = stats.total_freezes_duration_ms;
3319 info.total_pauses_duration_ms = stats.total_pauses_duration_ms;
3320
3321 info.content_type = stats.content_type;
3322
3323 info.firs_sent = stats.rtcp_packet_type_counts.fir_packets;
3324 info.plis_sent = stats.rtcp_packet_type_counts.pli_packets;
3325 info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets;
3326 // TODO(bugs.webrtc.org/10662): Add stats for LNTF.
3327
3328 info.timing_frame_info = stats.timing_frame_info;
3329
3330 if (log_stats)
3331 RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
3332
3333 return info;
3334 }
3335
3336 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetRecordableEncodedFrameCallback(std::function<void (const webrtc::RecordableEncodedFrame &)> callback)3337 SetRecordableEncodedFrameCallback(
3338 std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
3339 if (stream_) {
3340 stream_->SetAndGetRecordingState(
3341 webrtc::VideoReceiveStreamInterface::RecordingState(
3342 std::move(callback)),
3343 /*generate_key_frame=*/true);
3344 } else {
3345 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
3346 "frame sink";
3347 }
3348 }
3349
3350 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
ClearRecordableEncodedFrameCallback()3351 ClearRecordableEncodedFrameCallback() {
3352 if (stream_) {
3353 stream_->SetAndGetRecordingState(
3354 webrtc::VideoReceiveStreamInterface::RecordingState(),
3355 /*generate_key_frame=*/false);
3356 } else {
3357 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
3358 "frame sink";
3359 }
3360 }
3361
GenerateKeyFrame()3362 void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() {
3363 if (stream_) {
3364 stream_->GenerateKeyFrame();
3365 } else {
3366 RTC_LOG(LS_ERROR)
3367 << "Absent receive stream; ignoring key frame generation request.";
3368 }
3369 }
3370
3371 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3372 SetDepacketizerToDecoderFrameTransformer(
3373 rtc::scoped_refptr<webrtc::FrameTransformerInterface>
3374 frame_transformer) {
3375 config_.frame_transformer = frame_transformer;
3376 if (stream_)
3377 stream_->SetDepacketizerToDecoderFrameTransformer(frame_transformer);
3378 }
3379
SetLocalSsrc(uint32_t ssrc)3380 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetLocalSsrc(uint32_t ssrc) {
3381 config_.rtp.local_ssrc = ssrc;
3382 call_->OnLocalSsrcUpdated(stream(), ssrc);
3383 if (flexfec_stream_)
3384 call_->OnLocalSsrcUpdated(*flexfec_stream_, ssrc);
3385 }
3386
VideoCodecSettings()3387 WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings()
3388 : flexfec_payload_type(-1), rtx_payload_type(-1), rtx_time(-1) {}
3389
operator ==(const WebRtcVideoChannel::VideoCodecSettings & other) const3390 bool WebRtcVideoChannel::VideoCodecSettings::operator==(
3391 const WebRtcVideoChannel::VideoCodecSettings& other) const {
3392 return codec == other.codec && ulpfec == other.ulpfec &&
3393 flexfec_payload_type == other.flexfec_payload_type &&
3394 rtx_payload_type == other.rtx_payload_type &&
3395 rtx_time == other.rtx_time;
3396 }
3397
EqualsDisregardingFlexfec(const WebRtcVideoChannel::VideoCodecSettings & a,const WebRtcVideoChannel::VideoCodecSettings & b)3398 bool WebRtcVideoChannel::VideoCodecSettings::EqualsDisregardingFlexfec(
3399 const WebRtcVideoChannel::VideoCodecSettings& a,
3400 const WebRtcVideoChannel::VideoCodecSettings& b) {
3401 return a.codec == b.codec && a.ulpfec == b.ulpfec &&
3402 a.rtx_payload_type == b.rtx_payload_type && a.rtx_time == b.rtx_time;
3403 }
3404
operator !=(const WebRtcVideoChannel::VideoCodecSettings & other) const3405 bool WebRtcVideoChannel::VideoCodecSettings::operator!=(
3406 const WebRtcVideoChannel::VideoCodecSettings& other) const {
3407 return !(*this == other);
3408 }
3409
3410 std::vector<WebRtcVideoChannel::VideoCodecSettings>
MapCodecs(const std::vector<VideoCodec> & codecs)3411 WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) {
3412 if (codecs.empty()) {
3413 return {};
3414 }
3415
3416 std::vector<VideoCodecSettings> video_codecs;
3417 std::map<int, VideoCodec::CodecType> payload_codec_type;
3418 // `rtx_mapping` maps video payload type to rtx payload type.
3419 std::map<int, int> rtx_mapping;
3420 std::map<int, int> rtx_time_mapping;
3421
3422 webrtc::UlpfecConfig ulpfec_config;
3423 absl::optional<int> flexfec_payload_type;
3424
3425 for (const VideoCodec& in_codec : codecs) {
3426 const int payload_type = in_codec.id;
3427
3428 if (payload_codec_type.find(payload_type) != payload_codec_type.end()) {
3429 RTC_LOG(LS_ERROR) << "Payload type already registered: "
3430 << in_codec.ToString();
3431 return {};
3432 }
3433 payload_codec_type[payload_type] = in_codec.GetCodecType();
3434
3435 switch (in_codec.GetCodecType()) {
3436 case VideoCodec::CODEC_RED: {
3437 if (ulpfec_config.red_payload_type != -1) {
3438 RTC_LOG(LS_ERROR)
3439 << "Duplicate RED codec: ignoring PT=" << payload_type
3440 << " in favor of PT=" << ulpfec_config.red_payload_type
3441 << " which was specified first.";
3442 break;
3443 }
3444 ulpfec_config.red_payload_type = payload_type;
3445 break;
3446 }
3447
3448 case VideoCodec::CODEC_ULPFEC: {
3449 if (ulpfec_config.ulpfec_payload_type != -1) {
3450 RTC_LOG(LS_ERROR)
3451 << "Duplicate ULPFEC codec: ignoring PT=" << payload_type
3452 << " in favor of PT=" << ulpfec_config.ulpfec_payload_type
3453 << " which was specified first.";
3454 break;
3455 }
3456 ulpfec_config.ulpfec_payload_type = payload_type;
3457 break;
3458 }
3459
3460 case VideoCodec::CODEC_FLEXFEC: {
3461 if (flexfec_payload_type) {
3462 RTC_LOG(LS_ERROR)
3463 << "Duplicate FLEXFEC codec: ignoring PT=" << payload_type
3464 << " in favor of PT=" << *flexfec_payload_type
3465 << " which was specified first.";
3466 break;
3467 }
3468 flexfec_payload_type = payload_type;
3469 break;
3470 }
3471
3472 case VideoCodec::CODEC_RTX: {
3473 int associated_payload_type;
3474 if (!in_codec.GetParam(kCodecParamAssociatedPayloadType,
3475 &associated_payload_type) ||
3476 !IsValidRtpPayloadType(associated_payload_type)) {
3477 RTC_LOG(LS_ERROR)
3478 << "RTX codec with invalid or no associated payload type: "
3479 << in_codec.ToString();
3480 return {};
3481 }
3482 int rtx_time;
3483 if (in_codec.GetParam(kCodecParamRtxTime, &rtx_time) && rtx_time > 0) {
3484 rtx_time_mapping[associated_payload_type] = rtx_time;
3485 }
3486 rtx_mapping[associated_payload_type] = payload_type;
3487 break;
3488 }
3489
3490 case VideoCodec::CODEC_VIDEO: {
3491 video_codecs.emplace_back();
3492 video_codecs.back().codec = in_codec;
3493 break;
3494 }
3495 }
3496 }
3497
3498 // One of these codecs should have been a video codec. Only having FEC
3499 // parameters into this code is a logic error.
3500 RTC_DCHECK(!video_codecs.empty());
3501
3502 for (const auto& entry : rtx_mapping) {
3503 const int associated_payload_type = entry.first;
3504 const int rtx_payload_type = entry.second;
3505 auto it = payload_codec_type.find(associated_payload_type);
3506 if (it == payload_codec_type.end()) {
3507 RTC_LOG(LS_ERROR) << "RTX codec (PT=" << rtx_payload_type
3508 << ") mapped to PT=" << associated_payload_type
3509 << " which is not in the codec list.";
3510 return {};
3511 }
3512 const VideoCodec::CodecType associated_codec_type = it->second;
3513 if (associated_codec_type != VideoCodec::CODEC_VIDEO &&
3514 associated_codec_type != VideoCodec::CODEC_RED) {
3515 RTC_LOG(LS_ERROR)
3516 << "RTX PT=" << rtx_payload_type
3517 << " not mapped to regular video codec or RED codec (PT="
3518 << associated_payload_type << ").";
3519 return {};
3520 }
3521
3522 if (associated_payload_type == ulpfec_config.red_payload_type) {
3523 ulpfec_config.red_rtx_payload_type = rtx_payload_type;
3524 }
3525 }
3526
3527 for (VideoCodecSettings& codec_settings : video_codecs) {
3528 const int payload_type = codec_settings.codec.id;
3529 codec_settings.ulpfec = ulpfec_config;
3530 codec_settings.flexfec_payload_type = flexfec_payload_type.value_or(-1);
3531 auto it = rtx_mapping.find(payload_type);
3532 if (it != rtx_mapping.end()) {
3533 const int rtx_payload_type = it->second;
3534 codec_settings.rtx_payload_type = rtx_payload_type;
3535
3536 auto rtx_time_it = rtx_time_mapping.find(payload_type);
3537 if (rtx_time_it != rtx_time_mapping.end()) {
3538 const int rtx_time = rtx_time_it->second;
3539 if (rtx_time < kNackHistoryMs) {
3540 codec_settings.rtx_time = rtx_time;
3541 } else {
3542 codec_settings.rtx_time = kNackHistoryMs;
3543 }
3544 }
3545 }
3546 }
3547
3548 return video_codecs;
3549 }
3550
3551 WebRtcVideoChannel::WebRtcVideoReceiveStream*
FindReceiveStream(uint32_t ssrc)3552 WebRtcVideoChannel::FindReceiveStream(uint32_t ssrc) {
3553 if (ssrc == 0) {
3554 absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
3555 if (!default_ssrc) {
3556 return nullptr;
3557 }
3558 ssrc = *default_ssrc;
3559 }
3560 auto it = receive_streams_.find(ssrc);
3561 if (it != receive_streams_.end()) {
3562 return it->second;
3563 }
3564 return nullptr;
3565 }
3566
SetRecordableEncodedFrameCallback(uint32_t ssrc,std::function<void (const webrtc::RecordableEncodedFrame &)> callback)3567 void WebRtcVideoChannel::SetRecordableEncodedFrameCallback(
3568 uint32_t ssrc,
3569 std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
3570 RTC_DCHECK_RUN_ON(&thread_checker_);
3571 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3572 if (stream) {
3573 stream->SetRecordableEncodedFrameCallback(std::move(callback));
3574 } else {
3575 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
3576 "frame sink for ssrc "
3577 << ssrc;
3578 }
3579 }
3580
ClearRecordableEncodedFrameCallback(uint32_t ssrc)3581 void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
3582 RTC_DCHECK_RUN_ON(&thread_checker_);
3583 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3584 if (stream) {
3585 stream->ClearRecordableEncodedFrameCallback();
3586 } else {
3587 RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
3588 "frame sink for ssrc "
3589 << ssrc;
3590 }
3591 }
3592
RequestRecvKeyFrame(uint32_t ssrc)3593 void WebRtcVideoChannel::RequestRecvKeyFrame(uint32_t ssrc) {
3594 RTC_DCHECK_RUN_ON(&thread_checker_);
3595 WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3596 if (stream) {
3597 return stream->GenerateKeyFrame();
3598 } else {
3599 RTC_LOG(LS_ERROR)
3600 << "Absent receive stream; ignoring key frame generation for ssrc "
3601 << ssrc;
3602 }
3603 }
3604
GenerateSendKeyFrame(uint32_t ssrc,const std::vector<std::string> & rids)3605 void WebRtcVideoChannel::GenerateSendKeyFrame(
3606 uint32_t ssrc,
3607 const std::vector<std::string>& rids) {
3608 RTC_DCHECK_RUN_ON(&thread_checker_);
3609 auto it = send_streams_.find(ssrc);
3610 if (it != send_streams_.end()) {
3611 it->second->GenerateKeyFrame(rids);
3612 } else {
3613 RTC_LOG(LS_ERROR)
3614 << "Absent send stream; ignoring key frame generation for ssrc "
3615 << ssrc;
3616 }
3617 }
3618
SetEncoderToPacketizerFrameTransformer(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3619 void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
3620 uint32_t ssrc,
3621 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
3622 RTC_DCHECK_RUN_ON(&thread_checker_);
3623 auto matching_stream = send_streams_.find(ssrc);
3624 if (matching_stream != send_streams_.end()) {
3625 matching_stream->second->SetEncoderToPacketizerFrameTransformer(
3626 std::move(frame_transformer));
3627 }
3628 }
3629
SetDepacketizerToDecoderFrameTransformer(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3630 void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer(
3631 uint32_t ssrc,
3632 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
3633 RTC_DCHECK(frame_transformer);
3634 RTC_DCHECK_RUN_ON(&thread_checker_);
3635 if (ssrc == 0) {
3636 // If the receiver is unsignaled, save the frame transformer and set it when
3637 // the stream is associated with an ssrc.
3638 unsignaled_frame_transformer_ = std::move(frame_transformer);
3639 return;
3640 }
3641
3642 auto matching_stream = receive_streams_.find(ssrc);
3643 if (matching_stream != receive_streams_.end()) {
3644 matching_stream->second->SetDepacketizerToDecoderFrameTransformer(
3645 std::move(frame_transformer));
3646 }
3647 }
3648
3649 } // namespace cricket
3650