• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "media/engine/webrtc_video_engine.h"
12 
13 #include <stdio.h>
14 
15 #include <algorithm>
16 #include <set>
17 #include <string>
18 #include <utility>
19 
20 #include "absl/algorithm/container.h"
21 #include "absl/strings/match.h"
22 #include "api/media_stream_interface.h"
23 #include "api/video/video_codec_constants.h"
24 #include "api/video/video_codec_type.h"
25 #include "api/video_codecs/sdp_video_format.h"
26 #include "api/video_codecs/video_decoder_factory.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "api/video_codecs/video_encoder_factory.h"
29 #include "call/call.h"
30 #include "media/engine/webrtc_media_engine.h"
31 #include "media/engine/webrtc_voice_engine.h"
32 #include "modules/rtp_rtcp/source/rtp_util.h"
33 #include "modules/video_coding/codecs/vp9/svc_config.h"
34 #include "modules/video_coding/svc/scalability_mode_util.h"
35 #include "rtc_base/copy_on_write_buffer.h"
36 #include "rtc_base/experiments/field_trial_parser.h"
37 #include "rtc_base/experiments/field_trial_units.h"
38 #include "rtc_base/logging.h"
39 #include "rtc_base/numerics/safe_conversions.h"
40 #include "rtc_base/strings/string_builder.h"
41 #include "rtc_base/time_utils.h"
42 #include "rtc_base/trace_event.h"
43 
44 namespace cricket {
45 
46 namespace {
47 
48 using ::webrtc::ParseRtpPayloadType;
49 using ::webrtc::ParseRtpSsrc;
50 
51 constexpr int64_t kUnsignaledSsrcCooldownMs = rtc::kNumMillisecsPerSec / 2;
52 
53 // TODO(bugs.webrtc.org/13166): Remove AV1X when backwards compatibility is not
54 // needed.
55 constexpr char kAv1xCodecName[] = "AV1X";
56 
StreamTypeToString(webrtc::VideoSendStream::StreamStats::StreamType type)57 const char* StreamTypeToString(
58     webrtc::VideoSendStream::StreamStats::StreamType type) {
59   switch (type) {
60     case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
61       return "kMedia";
62     case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
63       return "kRtx";
64     case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
65       return "kFlexfec";
66   }
67   return nullptr;
68 }
69 
IsEnabled(const webrtc::FieldTrialsView & trials,absl::string_view name)70 bool IsEnabled(const webrtc::FieldTrialsView& trials, absl::string_view name) {
71   return absl::StartsWith(trials.Lookup(name), "Enabled");
72 }
73 
IsDisabled(const webrtc::FieldTrialsView & trials,absl::string_view name)74 bool IsDisabled(const webrtc::FieldTrialsView& trials, absl::string_view name) {
75   return absl::StartsWith(trials.Lookup(name), "Disabled");
76 }
77 
AddDefaultFeedbackParams(VideoCodec * codec,const webrtc::FieldTrialsView & trials)78 void AddDefaultFeedbackParams(VideoCodec* codec,
79                               const webrtc::FieldTrialsView& trials) {
80   // Don't add any feedback params for RED and ULPFEC.
81   if (codec->name == kRedCodecName || codec->name == kUlpfecCodecName)
82     return;
83   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty));
84   codec->AddFeedbackParam(
85       FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
86   // Don't add any more feedback params for FLEXFEC.
87   if (codec->name == kFlexfecCodecName)
88     return;
89   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir));
90   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty));
91   codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli));
92   if (codec->name == kVp8CodecName &&
93       IsEnabled(trials, "WebRTC-RtcpLossNotification")) {
94     codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamLntf, kParamValueEmpty));
95   }
96 }
97 
98 // Helper function to determine whether a codec should use the [35, 63] range.
99 // Should be used when adding new codecs (or variants).
IsCodecValidForLowerRange(const VideoCodec & codec)100 bool IsCodecValidForLowerRange(const VideoCodec& codec) {
101   if (absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName) ||
102       absl::EqualsIgnoreCase(codec.name, kAv1CodecName) ||
103       absl::EqualsIgnoreCase(codec.name, kAv1xCodecName)) {
104     return true;
105   } else if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) {
106     std::string profile_level_id;
107     std::string packetization_mode;
108 
109     if (codec.GetParam(kH264FmtpProfileLevelId, &profile_level_id)) {
110       if (absl::StartsWithIgnoreCase(profile_level_id, "4d00")) {
111         if (codec.GetParam(kH264FmtpPacketizationMode, &packetization_mode)) {
112           return packetization_mode == "0";
113         }
114       }
115       // H264 with YUV444.
116       return absl::StartsWithIgnoreCase(profile_level_id, "f400");
117     }
118   } else if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) {
119     std::string profile_id;
120 
121     if (codec.GetParam(kVP9ProfileId, &profile_id)) {
122       if (profile_id.compare("1") == 0 || profile_id.compare("3") == 0) {
123         return true;
124       }
125     }
126   }
127   return false;
128 }
129 
130 // This function will assign dynamic payload types (in the range [96, 127]
131 // and then [35, 63]) to the input codecs, and also add ULPFEC, RED, FlexFEC,
132 // and associated RTX codecs for recognized codecs (VP8, VP9, H264, and RED).
133 // It will also add default feedback params to the codecs.
134 // is_decoder_factory is needed to keep track of the implict assumption that any
135 // H264 decoder also supports constrained base line profile.
136 // Also, is_decoder_factory is used to decide whether FlexFEC video format
137 // should be advertised as supported.
138 // TODO(kron): Perhaps it is better to move the implicit knowledge to the place
139 // where codecs are negotiated.
140 template <class T>
GetPayloadTypesAndDefaultCodecs(const T * factory,bool is_decoder_factory,bool include_rtx,const webrtc::FieldTrialsView & trials)141 std::vector<VideoCodec> GetPayloadTypesAndDefaultCodecs(
142     const T* factory,
143     bool is_decoder_factory,
144     bool include_rtx,
145     const webrtc::FieldTrialsView& trials) {
146   if (!factory) {
147     return {};
148   }
149 
150   std::vector<webrtc::SdpVideoFormat> supported_formats =
151       factory->GetSupportedFormats();
152   if (is_decoder_factory) {
153     AddH264ConstrainedBaselineProfileToSupportedFormats(&supported_formats);
154   }
155 
156   if (supported_formats.empty())
157     return std::vector<VideoCodec>();
158 
159   supported_formats.push_back(webrtc::SdpVideoFormat(kRedCodecName));
160   supported_formats.push_back(webrtc::SdpVideoFormat(kUlpfecCodecName));
161 
162   // flexfec-03 is supported as
163   // - receive codec unless WebRTC-FlexFEC-03-Advertised is disabled
164   // - send codec if WebRTC-FlexFEC-03-Advertised is enabled
165   if ((is_decoder_factory &&
166        !IsDisabled(trials, "WebRTC-FlexFEC-03-Advertised")) ||
167       (!is_decoder_factory &&
168        IsEnabled(trials, "WebRTC-FlexFEC-03-Advertised"))) {
169     webrtc::SdpVideoFormat flexfec_format(kFlexfecCodecName);
170     // This value is currently arbitrarily set to 10 seconds. (The unit
171     // is microseconds.) This parameter MUST be present in the SDP, but
172     // we never use the actual value anywhere in our code however.
173     // TODO(brandtr): Consider honouring this value in the sender and receiver.
174     flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}};
175     supported_formats.push_back(flexfec_format);
176   }
177 
178   // Due to interoperability issues with old Chrome/WebRTC versions that
179   // ignore the [35, 63] range prefer the lower range for new codecs.
180   static const int kFirstDynamicPayloadTypeLowerRange = 35;
181   static const int kLastDynamicPayloadTypeLowerRange = 63;
182 
183   static const int kFirstDynamicPayloadTypeUpperRange = 96;
184   static const int kLastDynamicPayloadTypeUpperRange = 127;
185   int payload_type_upper = kFirstDynamicPayloadTypeUpperRange;
186   int payload_type_lower = kFirstDynamicPayloadTypeLowerRange;
187 
188   std::vector<VideoCodec> output_codecs;
189   for (const webrtc::SdpVideoFormat& format : supported_formats) {
190     VideoCodec codec(format);
191     bool isFecCodec = absl::EqualsIgnoreCase(codec.name, kUlpfecCodecName) ||
192                       absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName);
193 
194     // Check if we ran out of payload types.
195     if (payload_type_lower > kLastDynamicPayloadTypeLowerRange) {
196       // TODO(https://bugs.chromium.org/p/webrtc/issues/detail?id=12248):
197       // return an error.
198       RTC_LOG(LS_ERROR) << "Out of dynamic payload types [35,63] after "
199                            "fallback from [96, 127], skipping the rest.";
200       RTC_DCHECK_EQ(payload_type_upper, kLastDynamicPayloadTypeUpperRange);
201       break;
202     }
203 
204     // Lower range gets used for "new" codecs or when running out of payload
205     // types in the upper range.
206     if (IsCodecValidForLowerRange(codec) ||
207         payload_type_upper >= kLastDynamicPayloadTypeUpperRange) {
208       codec.id = payload_type_lower++;
209     } else {
210       codec.id = payload_type_upper++;
211     }
212     AddDefaultFeedbackParams(&codec, trials);
213     output_codecs.push_back(codec);
214 
215     // Add associated RTX codec for non-FEC codecs.
216     if (include_rtx) {
217       if (!isFecCodec) {
218         // Check if we ran out of payload types.
219         if (payload_type_lower > kLastDynamicPayloadTypeLowerRange) {
220           // TODO(https://bugs.chromium.org/p/webrtc/issues/detail?id=12248):
221           // return an error.
222           RTC_LOG(LS_ERROR) << "Out of dynamic payload types [35,63] after "
223                                "fallback from [96, 127], skipping the rest.";
224           RTC_DCHECK_EQ(payload_type_upper, kLastDynamicPayloadTypeUpperRange);
225           break;
226         }
227         if (IsCodecValidForLowerRange(codec) ||
228             payload_type_upper >= kLastDynamicPayloadTypeUpperRange) {
229           output_codecs.push_back(
230               VideoCodec::CreateRtxCodec(payload_type_lower++, codec.id));
231         } else {
232           output_codecs.push_back(
233               VideoCodec::CreateRtxCodec(payload_type_upper++, codec.id));
234         }
235       }
236     }
237   }
238   return output_codecs;
239 }
240 
CodecVectorToString(const std::vector<VideoCodec> & codecs)241 static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) {
242   rtc::StringBuilder out;
243   out << "{";
244   for (size_t i = 0; i < codecs.size(); ++i) {
245     out << codecs[i].ToString();
246     if (i != codecs.size() - 1) {
247       out << ", ";
248     }
249   }
250   out << "}";
251   return out.Release();
252 }
253 
ValidateCodecFormats(const std::vector<VideoCodec> & codecs)254 static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) {
255   bool has_video = false;
256   for (size_t i = 0; i < codecs.size(); ++i) {
257     if (!codecs[i].ValidateCodecFormat()) {
258       return false;
259     }
260     if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) {
261       has_video = true;
262     }
263   }
264   if (!has_video) {
265     RTC_LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: "
266                       << CodecVectorToString(codecs);
267     return false;
268   }
269   return true;
270 }
271 
ValidateStreamParams(const StreamParams & sp)272 static bool ValidateStreamParams(const StreamParams& sp) {
273   if (sp.ssrcs.empty()) {
274     RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString();
275     return false;
276   }
277 
278   std::vector<uint32_t> primary_ssrcs;
279   sp.GetPrimarySsrcs(&primary_ssrcs);
280   std::vector<uint32_t> rtx_ssrcs;
281   sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs);
282   for (uint32_t rtx_ssrc : rtx_ssrcs) {
283     bool rtx_ssrc_present = false;
284     for (uint32_t sp_ssrc : sp.ssrcs) {
285       if (sp_ssrc == rtx_ssrc) {
286         rtx_ssrc_present = true;
287         break;
288       }
289     }
290     if (!rtx_ssrc_present) {
291       RTC_LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc
292                         << "' missing from StreamParams ssrcs: "
293                         << sp.ToString();
294       return false;
295     }
296   }
297   if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) {
298     RTC_LOG(LS_ERROR)
299         << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): "
300         << sp.ToString();
301     return false;
302   }
303 
304   return true;
305 }
306 
307 // Returns true if the given codec is disallowed from doing simulcast.
IsCodecDisabledForSimulcast(const std::string & codec_name,const webrtc::FieldTrialsView & trials)308 bool IsCodecDisabledForSimulcast(const std::string& codec_name,
309                                  const webrtc::FieldTrialsView& trials) {
310   if (absl::EqualsIgnoreCase(codec_name, kVp9CodecName) ||
311       absl::EqualsIgnoreCase(codec_name, kAv1CodecName)) {
312     return true;
313   }
314 
315   if (absl::EqualsIgnoreCase(codec_name, kH264CodecName)) {
316     return absl::StartsWith(trials.Lookup("WebRTC-H264Simulcast"), "Disabled");
317   }
318 
319   return false;
320 }
321 
322 // Returns its smallest positive argument. If neither argument is positive,
323 // returns an arbitrary nonpositive value.
MinPositive(int a,int b)324 int MinPositive(int a, int b) {
325   if (a <= 0) {
326     return b;
327   }
328   if (b <= 0) {
329     return a;
330   }
331   return std::min(a, b);
332 }
333 
IsLayerActive(const webrtc::RtpEncodingParameters & layer)334 bool IsLayerActive(const webrtc::RtpEncodingParameters& layer) {
335   return layer.active &&
336          (!layer.max_bitrate_bps || *layer.max_bitrate_bps > 0) &&
337          (!layer.max_framerate || *layer.max_framerate > 0);
338 }
339 
NumActiveStreams(const webrtc::RtpParameters & rtp_parameters)340 int NumActiveStreams(const webrtc::RtpParameters& rtp_parameters) {
341   int res = 0;
342   for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) {
343     if (rtp_parameters.encodings[i].active) {
344       ++res;
345     }
346   }
347   return res;
348 }
349 
NumSpatialLayersFromEncoding(const webrtc::RtpParameters & rtp_parameters,size_t idx)350 absl::optional<int> NumSpatialLayersFromEncoding(
351     const webrtc::RtpParameters& rtp_parameters,
352     size_t idx) {
353   if (idx >= rtp_parameters.encodings.size())
354     return absl::nullopt;
355 
356   absl::optional<webrtc::ScalabilityMode> scalability_mode =
357       webrtc::ScalabilityModeFromString(
358           rtp_parameters.encodings[idx].scalability_mode.value_or(""));
359   return scalability_mode
360              ? absl::optional<int>(
361                    ScalabilityModeToNumSpatialLayers(*scalability_mode))
362              : absl::nullopt;
363 }
364 
365 std::map<uint32_t, webrtc::VideoSendStream::StreamStats>
MergeInfoAboutOutboundRtpSubstreams(const std::map<uint32_t,webrtc::VideoSendStream::StreamStats> & substreams)366 MergeInfoAboutOutboundRtpSubstreams(
367     const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>&
368         substreams) {
369   std::map<uint32_t, webrtc::VideoSendStream::StreamStats> rtp_substreams;
370   // Add substreams for all RTP media streams.
371   for (const auto& pair : substreams) {
372     uint32_t ssrc = pair.first;
373     const webrtc::VideoSendStream::StreamStats& substream = pair.second;
374     switch (substream.type) {
375       case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
376         break;
377       case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
378       case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
379         continue;
380     }
381     rtp_substreams.insert(std::make_pair(ssrc, substream));
382   }
383   // Complement the kMedia substream stats with the associated kRtx and kFlexfec
384   // substream stats.
385   for (const auto& pair : substreams) {
386     switch (pair.second.type) {
387       case webrtc::VideoSendStream::StreamStats::StreamType::kMedia:
388         continue;
389       case webrtc::VideoSendStream::StreamStats::StreamType::kRtx:
390       case webrtc::VideoSendStream::StreamStats::StreamType::kFlexfec:
391         break;
392     }
393     // The associated substream is an RTX or FlexFEC substream that is
394     // referencing an RTP media substream.
395     const webrtc::VideoSendStream::StreamStats& associated_substream =
396         pair.second;
397     RTC_DCHECK(associated_substream.referenced_media_ssrc.has_value());
398     uint32_t media_ssrc = associated_substream.referenced_media_ssrc.value();
399     if (substreams.find(media_ssrc) == substreams.end()) {
400       RTC_LOG(LS_WARNING) << "Substream [ssrc: " << pair.first << ", type: "
401                           << StreamTypeToString(associated_substream.type)
402                           << "] is associated with a media ssrc (" << media_ssrc
403                           << ") that does not have StreamStats. Ignoring its "
404                           << "RTP stats.";
405       continue;
406     }
407     webrtc::VideoSendStream::StreamStats& rtp_substream =
408         rtp_substreams[media_ssrc];
409 
410     // We only merge `rtp_stats`. All other metrics are not applicable for RTX
411     // and FlexFEC.
412     // TODO(hbos): kRtx and kFlexfec stats should use a separate struct to make
413     // it clear what is or is not applicable.
414     rtp_substream.rtp_stats.Add(associated_substream.rtp_stats);
415   }
416   return rtp_substreams;
417 }
418 
IsActiveFromEncodings(absl::optional<uint32_t> ssrc,const std::vector<webrtc::RtpEncodingParameters> & encodings)419 bool IsActiveFromEncodings(
420     absl::optional<uint32_t> ssrc,
421     const std::vector<webrtc::RtpEncodingParameters>& encodings) {
422   if (ssrc.has_value()) {
423     // Report the `active` value of a specific ssrc, or false if an encoding
424     // with this ssrc does not exist.
425     auto encoding_it = std::find_if(
426         encodings.begin(), encodings.end(),
427         [ssrc = ssrc.value()](const webrtc::RtpEncodingParameters& encoding) {
428           return encoding.ssrc.has_value() && encoding.ssrc.value() == ssrc;
429         });
430     return encoding_it != encodings.end() ? encoding_it->active : false;
431   }
432   // If `ssrc` is not specified then any encoding being active counts as active.
433   for (const auto& encoding : encodings) {
434     if (encoding.active) {
435       return true;
436     }
437   }
438   return false;
439 }
440 
441 }  // namespace
442 
443 // This constant is really an on/off, lower-level configurable NACK history
444 // duration hasn't been implemented.
445 static const int kNackHistoryMs = 1000;
446 
447 static const int kDefaultRtcpReceiverReportSsrc = 1;
448 
449 // Minimum time interval for logging stats.
450 static const int64_t kStatsLogIntervalMs = 10000;
451 
452 std::map<uint32_t, webrtc::VideoSendStream::StreamStats>
MergeInfoAboutOutboundRtpSubstreamsForTesting(const std::map<uint32_t,webrtc::VideoSendStream::StreamStats> & substreams)453 MergeInfoAboutOutboundRtpSubstreamsForTesting(
454     const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>&
455         substreams) {
456   return MergeInfoAboutOutboundRtpSubstreams(substreams);
457 }
458 
459 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
ConfigureVideoEncoderSettings(const VideoCodec & codec)460 WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings(
461     const VideoCodec& codec) {
462   RTC_DCHECK_RUN_ON(&thread_checker_);
463   bool is_screencast = parameters_.options.is_screencast.value_or(false);
464   // No automatic resizing when using simulcast or screencast, or when
465   // disabled by field trial flag.
466   bool automatic_resize = !disable_automatic_resize_ && !is_screencast &&
467                           (parameters_.config.rtp.ssrcs.size() == 1 ||
468                            NumActiveStreams(rtp_parameters_) == 1);
469 
470   bool denoising;
471   bool codec_default_denoising = false;
472   if (is_screencast) {
473     denoising = false;
474   } else {
475     // Use codec default if video_noise_reduction is unset.
476     codec_default_denoising = !parameters_.options.video_noise_reduction;
477     denoising = parameters_.options.video_noise_reduction.value_or(false);
478   }
479 
480   if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) {
481     return nullptr;
482   }
483   if (absl::EqualsIgnoreCase(codec.name, kVp8CodecName)) {
484     webrtc::VideoCodecVP8 vp8_settings =
485         webrtc::VideoEncoder::GetDefaultVp8Settings();
486     vp8_settings.automaticResizeOn = automatic_resize;
487     // VP8 denoising is enabled by default.
488     vp8_settings.denoisingOn = codec_default_denoising ? true : denoising;
489     return rtc::make_ref_counted<
490         webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
491   }
492   if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) {
493     webrtc::VideoCodecVP9 vp9_settings =
494         webrtc::VideoEncoder::GetDefaultVp9Settings();
495 
496     vp9_settings.numberOfSpatialLayers = std::min<unsigned char>(
497         parameters_.config.rtp.ssrcs.size(), kConferenceMaxNumSpatialLayers);
498     vp9_settings.numberOfTemporalLayers =
499         std::min<unsigned char>(parameters_.config.rtp.ssrcs.size() > 1
500                                     ? kConferenceDefaultNumTemporalLayers
501                                     : 1,
502                                 kConferenceMaxNumTemporalLayers);
503 
504     // VP9 denoising is disabled by default.
505     vp9_settings.denoisingOn = codec_default_denoising ? true : denoising;
506     // Disable automatic resize if more than one spatial layer is requested.
507     bool vp9_automatic_resize = automatic_resize;
508     absl::optional<int> num_spatial_layers =
509         NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0);
510     if (num_spatial_layers && *num_spatial_layers > 1) {
511       vp9_automatic_resize = false;
512     }
513     vp9_settings.automaticResizeOn = vp9_automatic_resize;
514     if (!is_screencast) {
515       webrtc::FieldTrialFlag interlayer_pred_experiment_enabled("Enabled");
516       webrtc::FieldTrialEnum<webrtc::InterLayerPredMode> inter_layer_pred_mode(
517           "inter_layer_pred_mode", webrtc::InterLayerPredMode::kOnKeyPic,
518           {{"off", webrtc::InterLayerPredMode::kOff},
519            {"on", webrtc::InterLayerPredMode::kOn},
520            {"onkeypic", webrtc::InterLayerPredMode::kOnKeyPic}});
521       webrtc::FieldTrialFlag force_flexible_mode("FlexibleMode");
522       webrtc::ParseFieldTrial(
523           {&interlayer_pred_experiment_enabled, &inter_layer_pred_mode,
524            &force_flexible_mode},
525           call_->trials().Lookup("WebRTC-Vp9InterLayerPred"));
526       if (interlayer_pred_experiment_enabled) {
527         vp9_settings.interLayerPred = inter_layer_pred_mode;
528       } else {
529         // Limit inter-layer prediction to key pictures by default.
530         vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic;
531       }
532       vp9_settings.flexibleMode = force_flexible_mode.Get();
533     } else {
534       // Multiple spatial layers vp9 screenshare needs flexible mode.
535       vp9_settings.flexibleMode = vp9_settings.numberOfSpatialLayers > 1;
536       vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOn;
537     }
538     return rtc::make_ref_counted<
539         webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
540   }
541   return nullptr;
542 }
543 
DefaultUnsignalledSsrcHandler()544 DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler()
545     : default_sink_(nullptr) {}
546 
OnUnsignalledSsrc(WebRtcVideoChannel * channel,uint32_t ssrc,absl::optional<uint32_t> rtx_ssrc)547 UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc(
548     WebRtcVideoChannel* channel,
549     uint32_t ssrc,
550     absl::optional<uint32_t> rtx_ssrc) {
551   absl::optional<uint32_t> default_recv_ssrc =
552       channel->GetDefaultReceiveStreamSsrc();
553 
554   if (default_recv_ssrc) {
555     RTC_LOG(LS_INFO) << "Destroying old default receive stream for SSRC="
556                      << ssrc << ".";
557     channel->RemoveRecvStream(*default_recv_ssrc);
558   }
559 
560   StreamParams sp = channel->unsignaled_stream_params();
561   sp.ssrcs.push_back(ssrc);
562   if (rtx_ssrc) {
563     sp.AddFidSsrc(ssrc, *rtx_ssrc);
564   }
565   RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc
566                    << ".";
567   if (!channel->AddRecvStream(sp, /*default_stream=*/true)) {
568     RTC_LOG(LS_WARNING) << "Could not create default receive stream.";
569   }
570 
571   // SSRC 0 returns default_recv_base_minimum_delay_ms.
572   const int unsignaled_ssrc = 0;
573   int default_recv_base_minimum_delay_ms =
574       channel->GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0);
575   // Set base minimum delay if it was set before for the default receive stream.
576   channel->SetBaseMinimumPlayoutDelayMs(ssrc,
577                                         default_recv_base_minimum_delay_ms);
578   channel->SetSink(ssrc, default_sink_);
579   return kDeliverPacket;
580 }
581 
582 rtc::VideoSinkInterface<webrtc::VideoFrame>*
GetDefaultSink() const583 DefaultUnsignalledSsrcHandler::GetDefaultSink() const {
584   return default_sink_;
585 }
586 
SetDefaultSink(WebRtcVideoChannel * channel,rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)587 void DefaultUnsignalledSsrcHandler::SetDefaultSink(
588     WebRtcVideoChannel* channel,
589     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
590   default_sink_ = sink;
591   absl::optional<uint32_t> default_recv_ssrc =
592       channel->GetDefaultReceiveStreamSsrc();
593   if (default_recv_ssrc) {
594     channel->SetSink(*default_recv_ssrc, default_sink_);
595   }
596 }
597 
WebRtcVideoEngine(std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,const webrtc::FieldTrialsView & trials)598 WebRtcVideoEngine::WebRtcVideoEngine(
599     std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
600     std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
601     const webrtc::FieldTrialsView& trials)
602     : decoder_factory_(std::move(video_decoder_factory)),
603       encoder_factory_(std::move(video_encoder_factory)),
604       trials_(trials) {
605   RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()";
606 }
607 
~WebRtcVideoEngine()608 WebRtcVideoEngine::~WebRtcVideoEngine() {
609   RTC_DLOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
610 }
611 
CreateMediaChannel(webrtc::Call * call,const MediaConfig & config,const VideoOptions & options,const webrtc::CryptoOptions & crypto_options,webrtc::VideoBitrateAllocatorFactory * video_bitrate_allocator_factory)612 VideoMediaChannel* WebRtcVideoEngine::CreateMediaChannel(
613     webrtc::Call* call,
614     const MediaConfig& config,
615     const VideoOptions& options,
616     const webrtc::CryptoOptions& crypto_options,
617     webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) {
618   RTC_LOG(LS_INFO) << "CreateMediaChannel. Options: " << options.ToString();
619   return new WebRtcVideoChannel(call, config, options, crypto_options,
620                                 encoder_factory_.get(), decoder_factory_.get(),
621                                 video_bitrate_allocator_factory);
622 }
send_codecs(bool include_rtx) const623 std::vector<VideoCodec> WebRtcVideoEngine::send_codecs(bool include_rtx) const {
624   return GetPayloadTypesAndDefaultCodecs(encoder_factory_.get(),
625                                          /*is_decoder_factory=*/false,
626                                          include_rtx, trials_);
627 }
628 
recv_codecs(bool include_rtx) const629 std::vector<VideoCodec> WebRtcVideoEngine::recv_codecs(bool include_rtx) const {
630   return GetPayloadTypesAndDefaultCodecs(decoder_factory_.get(),
631                                          /*is_decoder_factory=*/true,
632                                          include_rtx, trials_);
633 }
634 
635 std::vector<webrtc::RtpHeaderExtensionCapability>
GetRtpHeaderExtensions() const636 WebRtcVideoEngine::GetRtpHeaderExtensions() const {
637   std::vector<webrtc::RtpHeaderExtensionCapability> result;
638   int id = 1;
639   for (const auto& uri :
640        {webrtc::RtpExtension::kTimestampOffsetUri,
641         webrtc::RtpExtension::kAbsSendTimeUri,
642         webrtc::RtpExtension::kVideoRotationUri,
643         webrtc::RtpExtension::kTransportSequenceNumberUri,
644         webrtc::RtpExtension::kPlayoutDelayUri,
645         webrtc::RtpExtension::kVideoContentTypeUri,
646         webrtc::RtpExtension::kVideoTimingUri,
647         webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri,
648         webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) {
649     result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv);
650   }
651   result.emplace_back(webrtc::RtpExtension::kGenericFrameDescriptorUri00, id++,
652                       IsEnabled(trials_, "WebRTC-GenericDescriptorAdvertised")
653                           ? webrtc::RtpTransceiverDirection::kSendRecv
654                           : webrtc::RtpTransceiverDirection::kStopped);
655   result.emplace_back(
656       webrtc::RtpExtension::kDependencyDescriptorUri, id++,
657       IsEnabled(trials_, "WebRTC-DependencyDescriptorAdvertised")
658           ? webrtc::RtpTransceiverDirection::kSendRecv
659           : webrtc::RtpTransceiverDirection::kStopped);
660 
661   result.emplace_back(
662       webrtc::RtpExtension::kVideoLayersAllocationUri, id++,
663       IsEnabled(trials_, "WebRTC-VideoLayersAllocationAdvertised")
664           ? webrtc::RtpTransceiverDirection::kSendRecv
665           : webrtc::RtpTransceiverDirection::kStopped);
666 
667   result.emplace_back(
668       webrtc::RtpExtension::kVideoFrameTrackingIdUri, id++,
669       IsEnabled(trials_, "WebRTC-VideoFrameTrackingIdAdvertised")
670           ? webrtc::RtpTransceiverDirection::kSendRecv
671           : webrtc::RtpTransceiverDirection::kStopped);
672 
673   return result;
674 }
675 
WebRtcVideoChannel(webrtc::Call * call,const MediaConfig & config,const VideoOptions & options,const webrtc::CryptoOptions & crypto_options,webrtc::VideoEncoderFactory * encoder_factory,webrtc::VideoDecoderFactory * decoder_factory,webrtc::VideoBitrateAllocatorFactory * bitrate_allocator_factory)676 WebRtcVideoChannel::WebRtcVideoChannel(
677     webrtc::Call* call,
678     const MediaConfig& config,
679     const VideoOptions& options,
680     const webrtc::CryptoOptions& crypto_options,
681     webrtc::VideoEncoderFactory* encoder_factory,
682     webrtc::VideoDecoderFactory* decoder_factory,
683     webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
684     : VideoMediaChannel(call->network_thread(), config.enable_dscp),
685       worker_thread_(call->worker_thread()),
686       call_(call),
687       unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
688       video_config_(config.video),
689       encoder_factory_(encoder_factory),
690       decoder_factory_(decoder_factory),
691       bitrate_allocator_factory_(bitrate_allocator_factory),
692       default_send_options_(options),
693       last_stats_log_ms_(-1),
694       discard_unknown_ssrc_packets_(
695           IsEnabled(call_->trials(),
696                     "WebRTC-Video-DiscardPacketsWithUnknownSsrc")),
697       crypto_options_(crypto_options),
698       unknown_ssrc_packet_buffer_(
699           IsEnabled(call_->trials(),
700                     "WebRTC-Video-BufferPacketsWithUnknownSsrc")
701               ? new UnhandledPacketsBuffer()
702               : nullptr) {
703   RTC_DCHECK_RUN_ON(&thread_checker_);
704   network_thread_checker_.Detach();
705 
706   rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc;
707   sending_ = false;
708   recv_codecs_ = MapCodecs(GetPayloadTypesAndDefaultCodecs(
709       decoder_factory_, /*is_decoder_factory=*/true,
710       /*include_rtx=*/true, call_->trials()));
711   recv_flexfec_payload_type_ =
712       recv_codecs_.empty() ? 0 : recv_codecs_.front().flexfec_payload_type;
713 }
714 
~WebRtcVideoChannel()715 WebRtcVideoChannel::~WebRtcVideoChannel() {
716   for (auto& kv : send_streams_)
717     delete kv.second;
718   for (auto& kv : receive_streams_)
719     delete kv.second;
720 }
721 
722 std::vector<WebRtcVideoChannel::VideoCodecSettings>
SelectSendVideoCodecs(const std::vector<VideoCodecSettings> & remote_mapped_codecs) const723 WebRtcVideoChannel::SelectSendVideoCodecs(
724     const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
725   std::vector<webrtc::SdpVideoFormat> sdp_formats =
726       encoder_factory_ ? encoder_factory_->GetImplementations()
727                        : std::vector<webrtc::SdpVideoFormat>();
728 
729   // The returned vector holds the VideoCodecSettings in term of preference.
730   // They are orderd by receive codec preference first and local implementation
731   // preference second.
732   std::vector<VideoCodecSettings> encoders;
733   for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) {
734     for (auto format_it = sdp_formats.begin();
735          format_it != sdp_formats.end();) {
736       // For H264, we will limit the encode level to the remote offered level
737       // regardless if level asymmetry is allowed or not. This is strictly not
738       // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
739       // since we should limit the encode level to the lower of local and remote
740       // level when level asymmetry is not allowed.
741       if (format_it->IsSameCodec(
742               {remote_codec.codec.name, remote_codec.codec.params})) {
743         encoders.push_back(remote_codec);
744 
745         // To allow the VideoEncoderFactory to keep information about which
746         // implementation to instantitate when CreateEncoder is called the two
747         // parmeter sets are merged.
748         encoders.back().codec.params.insert(format_it->parameters.begin(),
749                                             format_it->parameters.end());
750 
751         format_it = sdp_formats.erase(format_it);
752       } else {
753         ++format_it;
754       }
755     }
756   }
757 
758   return encoders;
759 }
760 
NonFlexfecReceiveCodecsHaveChanged(std::vector<VideoCodecSettings> before,std::vector<VideoCodecSettings> after)761 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
762     std::vector<VideoCodecSettings> before,
763     std::vector<VideoCodecSettings> after) {
764   // The receive codec order doesn't matter, so we sort the codecs before
765   // comparing. This is necessary because currently the
766   // only way to change the send codec is to munge SDP, which causes
767   // the receive codec list to change order, which causes the streams
768   // to be recreates which causes a "blink" of black video.  In order
769   // to support munging the SDP in this way without recreating receive
770   // streams, we ignore the order of the received codecs so that
771   // changing the order doesn't cause this "blink".
772   auto comparison = [](const VideoCodecSettings& codec1,
773                        const VideoCodecSettings& codec2) {
774     return codec1.codec.id > codec2.codec.id;
775   };
776   absl::c_sort(before, comparison);
777   absl::c_sort(after, comparison);
778 
779   // Changes in FlexFEC payload type are handled separately in
780   // WebRtcVideoChannel::GetChangedRecvParameters, so disregard FlexFEC in the
781   // comparison here.
782   return !absl::c_equal(before, after,
783                         VideoCodecSettings::EqualsDisregardingFlexfec);
784 }
785 
GetChangedSendParameters(const VideoSendParameters & params,ChangedSendParameters * changed_params) const786 bool WebRtcVideoChannel::GetChangedSendParameters(
787     const VideoSendParameters& params,
788     ChangedSendParameters* changed_params) const {
789   if (!ValidateCodecFormats(params.codecs) ||
790       !ValidateRtpExtensions(params.extensions, send_rtp_extensions_)) {
791     return false;
792   }
793 
794   std::vector<VideoCodecSettings> negotiated_codecs =
795       SelectSendVideoCodecs(MapCodecs(params.codecs));
796 
797   // We should only fail here if send direction is enabled.
798   if (params.is_stream_active && negotiated_codecs.empty()) {
799     RTC_LOG(LS_ERROR) << "No video codecs supported.";
800     return false;
801   }
802 
803   // Never enable sending FlexFEC, unless we are in the experiment.
804   if (!IsEnabled(call_->trials(), "WebRTC-FlexFEC-03")) {
805     for (VideoCodecSettings& codec : negotiated_codecs)
806       codec.flexfec_payload_type = -1;
807   }
808 
809   if (negotiated_codecs_ != negotiated_codecs) {
810     if (negotiated_codecs.empty()) {
811       changed_params->send_codec = absl::nullopt;
812     } else if (send_codec_ != negotiated_codecs.front()) {
813       changed_params->send_codec = negotiated_codecs.front();
814     }
815     changed_params->negotiated_codecs = std::move(negotiated_codecs);
816   }
817 
818   // Handle RTP header extensions.
819   if (params.extmap_allow_mixed != ExtmapAllowMixed()) {
820     changed_params->extmap_allow_mixed = params.extmap_allow_mixed;
821   }
822   std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
823       params.extensions, webrtc::RtpExtension::IsSupportedForVideo, true,
824       call_->trials());
825   if (send_rtp_extensions_ != filtered_extensions) {
826     changed_params->rtp_header_extensions =
827         absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
828   }
829 
830   if (params.mid != send_params_.mid) {
831     changed_params->mid = params.mid;
832   }
833 
834   // Handle max bitrate.
835   if (params.max_bandwidth_bps != send_params_.max_bandwidth_bps &&
836       params.max_bandwidth_bps >= -1) {
837     // 0 or -1 uncaps max bitrate.
838     // TODO(pbos): Reconsider how 0 should be treated. It is not mentioned as a
839     // special value and might very well be used for stopping sending.
840     changed_params->max_bandwidth_bps =
841         params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps;
842   }
843 
844   // Handle conference mode.
845   if (params.conference_mode != send_params_.conference_mode) {
846     changed_params->conference_mode = params.conference_mode;
847   }
848 
849   // Handle RTCP mode.
850   if (params.rtcp.reduced_size != send_params_.rtcp.reduced_size) {
851     changed_params->rtcp_mode = params.rtcp.reduced_size
852                                     ? webrtc::RtcpMode::kReducedSize
853                                     : webrtc::RtcpMode::kCompound;
854   }
855 
856   return true;
857 }
858 
SetSendParameters(const VideoSendParameters & params)859 bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
860   RTC_DCHECK_RUN_ON(&thread_checker_);
861   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSendParameters");
862   RTC_LOG(LS_INFO) << "SetSendParameters: " << params.ToString();
863   ChangedSendParameters changed_params;
864   if (!GetChangedSendParameters(params, &changed_params)) {
865     return false;
866   }
867 
868   if (changed_params.negotiated_codecs) {
869     for (const auto& send_codec : *changed_params.negotiated_codecs)
870       RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString();
871   }
872 
873   send_params_ = params;
874   return ApplyChangedParams(changed_params);
875 }
876 
RequestEncoderFallback()877 void WebRtcVideoChannel::RequestEncoderFallback() {
878   if (!worker_thread_->IsCurrent()) {
879     worker_thread_->PostTask(
880         SafeTask(task_safety_.flag(), [this] { RequestEncoderFallback(); }));
881     return;
882   }
883 
884   RTC_DCHECK_RUN_ON(&thread_checker_);
885   if (negotiated_codecs_.size() <= 1) {
886     RTC_LOG(LS_WARNING) << "Encoder failed but no fallback codec is available";
887     return;
888   }
889 
890   ChangedSendParameters params;
891   params.negotiated_codecs = negotiated_codecs_;
892   params.negotiated_codecs->erase(params.negotiated_codecs->begin());
893   params.send_codec = params.negotiated_codecs->front();
894   ApplyChangedParams(params);
895 }
896 
RequestEncoderSwitch(const webrtc::SdpVideoFormat & format,bool allow_default_fallback)897 void WebRtcVideoChannel::RequestEncoderSwitch(
898     const webrtc::SdpVideoFormat& format,
899     bool allow_default_fallback) {
900   if (!worker_thread_->IsCurrent()) {
901     worker_thread_->PostTask(
902         SafeTask(task_safety_.flag(), [this, format, allow_default_fallback] {
903           RequestEncoderSwitch(format, allow_default_fallback);
904         }));
905     return;
906   }
907 
908   RTC_DCHECK_RUN_ON(&thread_checker_);
909 
910   for (const VideoCodecSettings& codec_setting : negotiated_codecs_) {
911     if (format.IsSameCodec(
912             {codec_setting.codec.name, codec_setting.codec.params})) {
913       VideoCodecSettings new_codec_setting = codec_setting;
914       for (const auto& kv : format.parameters) {
915         new_codec_setting.codec.params[kv.first] = kv.second;
916       }
917 
918       if (send_codec_ == new_codec_setting) {
919         // Already using this codec, no switch required.
920         return;
921       }
922 
923       ChangedSendParameters params;
924       params.send_codec = new_codec_setting;
925       ApplyChangedParams(params);
926       return;
927     }
928   }
929 
930   RTC_LOG(LS_WARNING) << "Failed to switch encoder to: " << format.ToString()
931                       << ". Is default fallback allowed: "
932                       << allow_default_fallback;
933 
934   if (allow_default_fallback) {
935     RequestEncoderFallback();
936   }
937 }
938 
ApplyChangedParams(const ChangedSendParameters & changed_params)939 bool WebRtcVideoChannel::ApplyChangedParams(
940     const ChangedSendParameters& changed_params) {
941   RTC_DCHECK_RUN_ON(&thread_checker_);
942   if (changed_params.negotiated_codecs)
943     negotiated_codecs_ = *changed_params.negotiated_codecs;
944 
945   if (changed_params.send_codec)
946     send_codec_ = changed_params.send_codec;
947 
948   if (changed_params.extmap_allow_mixed) {
949     SetExtmapAllowMixed(*changed_params.extmap_allow_mixed);
950   }
951   if (changed_params.rtp_header_extensions) {
952     send_rtp_extensions_ = *changed_params.rtp_header_extensions;
953   }
954 
955   if (changed_params.send_codec || changed_params.max_bandwidth_bps) {
956     if (send_params_.max_bandwidth_bps == -1) {
957       // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is
958       // -1, which corresponds to no "b=AS" attribute in SDP. Note that the
959       // global max bitrate may be set below in GetBitrateConfigForCodec, from
960       // the codec max bitrate.
961       // TODO(pbos): This should be reconsidered (codec max bitrate should
962       // probably not affect global call max bitrate).
963       bitrate_config_.max_bitrate_bps = -1;
964     }
965 
966     if (send_codec_) {
967       // TODO(holmer): Changing the codec parameters shouldn't necessarily mean
968       // that we change the min/max of bandwidth estimation. Reevaluate this.
969       bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec);
970       if (!changed_params.send_codec) {
971         // If the codec isn't changing, set the start bitrate to -1 which means
972         // "unchanged" so that BWE isn't affected.
973         bitrate_config_.start_bitrate_bps = -1;
974       }
975     }
976 
977     if (send_params_.max_bandwidth_bps >= 0) {
978       // Note that max_bandwidth_bps intentionally takes priority over the
979       // bitrate config for the codec. This allows FEC to be applied above the
980       // codec target bitrate.
981       // TODO(pbos): Figure out whether b=AS means max bitrate for this
982       // WebRtcVideoChannel (in which case we're good), or per sender (SSRC),
983       // in which case this should not set a BitrateConstraints but rather
984       // reconfigure all senders.
985       bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0
986                                             ? -1
987                                             : send_params_.max_bandwidth_bps;
988     }
989 
990     call_->GetTransportControllerSend()->SetSdpBitrateParameters(
991         bitrate_config_);
992   }
993 
994   for (auto& kv : send_streams_) {
995     kv.second->SetSendParameters(changed_params);
996   }
997   if (changed_params.send_codec || changed_params.rtcp_mode) {
998     // Update receive feedback parameters from new codec or RTCP mode.
999     RTC_LOG(LS_INFO)
1000         << "SetFeedbackParameters on all the receive streams because the send "
1001            "codec or RTCP mode has changed.";
1002     for (auto& kv : receive_streams_) {
1003       RTC_DCHECK(kv.second != nullptr);
1004       kv.second->SetFeedbackParameters(
1005           HasLntf(send_codec_->codec), HasNack(send_codec_->codec),
1006           HasTransportCc(send_codec_->codec),
1007           send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
1008                                          : webrtc::RtcpMode::kCompound,
1009           send_codec_->rtx_time);
1010     }
1011   }
1012   return true;
1013 }
1014 
GetRtpSendParameters(uint32_t ssrc) const1015 webrtc::RtpParameters WebRtcVideoChannel::GetRtpSendParameters(
1016     uint32_t ssrc) const {
1017   RTC_DCHECK_RUN_ON(&thread_checker_);
1018   auto it = send_streams_.find(ssrc);
1019   if (it == send_streams_.end()) {
1020     RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream "
1021                            "with ssrc "
1022                         << ssrc << " which doesn't exist.";
1023     return webrtc::RtpParameters();
1024   }
1025 
1026   webrtc::RtpParameters rtp_params = it->second->GetRtpParameters();
1027   // Need to add the common list of codecs to the send stream-specific
1028   // RTP parameters.
1029   for (const VideoCodec& codec : send_params_.codecs) {
1030     if (send_codec_ && send_codec_->codec.id == codec.id) {
1031       // Put the current send codec to the front of the codecs list.
1032       RTC_DCHECK_EQ(codec.name, send_codec_->codec.name);
1033       rtp_params.codecs.insert(rtp_params.codecs.begin(),
1034                                codec.ToCodecParameters());
1035     } else {
1036       rtp_params.codecs.push_back(codec.ToCodecParameters());
1037     }
1038   }
1039 
1040   return rtp_params;
1041 }
1042 
SetRtpSendParameters(uint32_t ssrc,const webrtc::RtpParameters & parameters,webrtc::SetParametersCallback callback)1043 webrtc::RTCError WebRtcVideoChannel::SetRtpSendParameters(
1044     uint32_t ssrc,
1045     const webrtc::RtpParameters& parameters,
1046     webrtc::SetParametersCallback callback) {
1047   RTC_DCHECK_RUN_ON(&thread_checker_);
1048   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRtpSendParameters");
1049   auto it = send_streams_.find(ssrc);
1050   if (it == send_streams_.end()) {
1051     RTC_LOG(LS_ERROR) << "Attempting to set RTP send parameters for stream "
1052                          "with ssrc "
1053                       << ssrc << " which doesn't exist.";
1054     return webrtc::InvokeSetParametersCallback(
1055         callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR));
1056   }
1057 
1058   // TODO(deadbeef): Handle setting parameters with a list of codecs in a
1059   // different order (which should change the send codec).
1060   webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc);
1061   if (current_parameters.codecs != parameters.codecs) {
1062     RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs "
1063                           "is not currently supported.";
1064     return webrtc::InvokeSetParametersCallback(
1065         callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR));
1066   }
1067 
1068   if (!parameters.encodings.empty()) {
1069     // Note that these values come from:
1070     // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5
1071     // TODO(deadbeef): Change values depending on whether we are sending a
1072     // keyframe or non-keyframe.
1073     rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT;
1074     switch (parameters.encodings[0].network_priority) {
1075       case webrtc::Priority::kVeryLow:
1076         new_dscp = rtc::DSCP_CS1;
1077         break;
1078       case webrtc::Priority::kLow:
1079         new_dscp = rtc::DSCP_DEFAULT;
1080         break;
1081       case webrtc::Priority::kMedium:
1082         new_dscp = rtc::DSCP_AF42;
1083         break;
1084       case webrtc::Priority::kHigh:
1085         new_dscp = rtc::DSCP_AF41;
1086         break;
1087     }
1088     SetPreferredDscp(new_dscp);
1089   }
1090 
1091   return it->second->SetRtpParameters(parameters, std::move(callback));
1092 }
1093 
GetRtpReceiveParameters(uint32_t ssrc) const1094 webrtc::RtpParameters WebRtcVideoChannel::GetRtpReceiveParameters(
1095     uint32_t ssrc) const {
1096   RTC_DCHECK_RUN_ON(&thread_checker_);
1097   webrtc::RtpParameters rtp_params;
1098   auto it = receive_streams_.find(ssrc);
1099   if (it == receive_streams_.end()) {
1100     RTC_LOG(LS_WARNING)
1101         << "Attempting to get RTP receive parameters for stream "
1102            "with SSRC "
1103         << ssrc << " which doesn't exist.";
1104     return webrtc::RtpParameters();
1105   }
1106   rtp_params = it->second->GetRtpParameters();
1107 
1108   // Add codecs, which any stream is prepared to receive.
1109   for (const VideoCodec& codec : recv_params_.codecs) {
1110     rtp_params.codecs.push_back(codec.ToCodecParameters());
1111   }
1112 
1113   return rtp_params;
1114 }
1115 
GetDefaultRtpReceiveParameters() const1116 webrtc::RtpParameters WebRtcVideoChannel::GetDefaultRtpReceiveParameters()
1117     const {
1118   RTC_DCHECK_RUN_ON(&thread_checker_);
1119   webrtc::RtpParameters rtp_params;
1120   if (!default_unsignalled_ssrc_handler_.GetDefaultSink()) {
1121     // Getting parameters on a default, unsignaled video receive stream but
1122     // because we've not configured to receive such a stream, `encodings` is
1123     // empty.
1124     return rtp_params;
1125   }
1126   rtp_params.encodings.emplace_back();
1127 
1128   // Add codecs, which any stream is prepared to receive.
1129   for (const VideoCodec& codec : recv_params_.codecs) {
1130     rtp_params.codecs.push_back(codec.ToCodecParameters());
1131   }
1132 
1133   return rtp_params;
1134 }
1135 
GetChangedRecvParameters(const VideoRecvParameters & params,ChangedRecvParameters * changed_params) const1136 bool WebRtcVideoChannel::GetChangedRecvParameters(
1137     const VideoRecvParameters& params,
1138     ChangedRecvParameters* changed_params) const {
1139   if (!ValidateCodecFormats(params.codecs) ||
1140       !ValidateRtpExtensions(params.extensions, recv_rtp_extensions_)) {
1141     return false;
1142   }
1143 
1144   // Handle receive codecs.
1145   const std::vector<VideoCodecSettings> mapped_codecs =
1146       MapCodecs(params.codecs);
1147   if (mapped_codecs.empty()) {
1148     RTC_LOG(LS_ERROR)
1149         << "GetChangedRecvParameters called without any video codecs.";
1150     return false;
1151   }
1152 
1153   // Verify that every mapped codec is supported locally.
1154   if (params.is_stream_active) {
1155     const std::vector<VideoCodec> local_supported_codecs =
1156         GetPayloadTypesAndDefaultCodecs(decoder_factory_,
1157                                         /*is_decoder_factory=*/true,
1158                                         /*include_rtx=*/true, call_->trials());
1159     for (const VideoCodecSettings& mapped_codec : mapped_codecs) {
1160       if (!FindMatchingCodec(local_supported_codecs, mapped_codec.codec)) {
1161         RTC_LOG(LS_ERROR)
1162             << "GetChangedRecvParameters called with unsupported video codec: "
1163             << mapped_codec.codec.ToString();
1164         return false;
1165       }
1166     }
1167   }
1168 
1169   if (NonFlexfecReceiveCodecsHaveChanged(recv_codecs_, mapped_codecs)) {
1170     changed_params->codec_settings =
1171         absl::optional<std::vector<VideoCodecSettings>>(mapped_codecs);
1172   }
1173 
1174   // Handle RTP header extensions.
1175   std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
1176       params.extensions, webrtc::RtpExtension::IsSupportedForVideo, false,
1177       call_->trials());
1178   if (filtered_extensions != recv_rtp_extensions_) {
1179     changed_params->rtp_header_extensions =
1180         absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
1181   }
1182 
1183   int flexfec_payload_type = mapped_codecs.front().flexfec_payload_type;
1184   if (flexfec_payload_type != recv_flexfec_payload_type_) {
1185     changed_params->flexfec_payload_type = flexfec_payload_type;
1186   }
1187 
1188   return true;
1189 }
1190 
SetRecvParameters(const VideoRecvParameters & params)1191 bool WebRtcVideoChannel::SetRecvParameters(const VideoRecvParameters& params) {
1192   RTC_DCHECK_RUN_ON(&thread_checker_);
1193   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRecvParameters");
1194   RTC_LOG(LS_INFO) << "SetRecvParameters: " << params.ToString();
1195   ChangedRecvParameters changed_params;
1196   if (!GetChangedRecvParameters(params, &changed_params)) {
1197     return false;
1198   }
1199   if (changed_params.flexfec_payload_type) {
1200     RTC_DLOG(LS_INFO) << "Changing FlexFEC payload type (recv) from "
1201                       << recv_flexfec_payload_type_ << " to "
1202                       << *changed_params.flexfec_payload_type;
1203     recv_flexfec_payload_type_ = *changed_params.flexfec_payload_type;
1204   }
1205   if (changed_params.rtp_header_extensions) {
1206     recv_rtp_extensions_ = *changed_params.rtp_header_extensions;
1207   }
1208   if (changed_params.codec_settings) {
1209     RTC_DLOG(LS_INFO) << "Changing recv codecs from "
1210                       << CodecSettingsVectorToString(recv_codecs_) << " to "
1211                       << CodecSettingsVectorToString(
1212                              *changed_params.codec_settings);
1213     recv_codecs_ = *changed_params.codec_settings;
1214   }
1215 
1216   for (auto& kv : receive_streams_) {
1217     kv.second->SetRecvParameters(changed_params);
1218   }
1219   recv_params_ = params;
1220   return true;
1221 }
1222 
CodecSettingsVectorToString(const std::vector<VideoCodecSettings> & codecs)1223 std::string WebRtcVideoChannel::CodecSettingsVectorToString(
1224     const std::vector<VideoCodecSettings>& codecs) {
1225   rtc::StringBuilder out;
1226   out << "{";
1227   for (size_t i = 0; i < codecs.size(); ++i) {
1228     out << codecs[i].codec.ToString();
1229     if (i != codecs.size() - 1) {
1230       out << ", ";
1231     }
1232   }
1233   out << "}";
1234   return out.Release();
1235 }
1236 
ExtractCodecInformation(rtc::ArrayView<const VideoCodecSettings> recv_codecs,std::map<int,int> & rtx_associated_payload_types,std::set<int> & raw_payload_types,std::vector<webrtc::VideoReceiveStreamInterface::Decoder> & decoders)1237 void WebRtcVideoChannel::ExtractCodecInformation(
1238     rtc::ArrayView<const VideoCodecSettings> recv_codecs,
1239     std::map<int, int>& rtx_associated_payload_types,
1240     std::set<int>& raw_payload_types,
1241     std::vector<webrtc::VideoReceiveStreamInterface::Decoder>& decoders) {
1242   RTC_DCHECK(!recv_codecs.empty());
1243   RTC_DCHECK(rtx_associated_payload_types.empty());
1244   RTC_DCHECK(raw_payload_types.empty());
1245   RTC_DCHECK(decoders.empty());
1246 
1247   for (const VideoCodecSettings& recv_codec : recv_codecs) {
1248     decoders.emplace_back(
1249         webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params),
1250         recv_codec.codec.id);
1251     rtx_associated_payload_types.emplace(recv_codec.rtx_payload_type,
1252                                          recv_codec.codec.id);
1253     if (recv_codec.codec.packetization == kPacketizationParamRaw) {
1254       raw_payload_types.insert(recv_codec.codec.id);
1255     }
1256   }
1257 }
1258 
SetReceiverReportSsrc(uint32_t ssrc)1259 void WebRtcVideoChannel::SetReceiverReportSsrc(uint32_t ssrc) {
1260   RTC_DCHECK_RUN_ON(&thread_checker_);
1261   if (ssrc == rtcp_receiver_report_ssrc_)
1262     return;
1263 
1264   rtcp_receiver_report_ssrc_ = ssrc;
1265   for (auto& [unused, receive_stream] : receive_streams_)
1266     receive_stream->SetLocalSsrc(ssrc);
1267 }
1268 
GetSendCodec(VideoCodec * codec)1269 bool WebRtcVideoChannel::GetSendCodec(VideoCodec* codec) {
1270   RTC_DCHECK_RUN_ON(&thread_checker_);
1271   if (!send_codec_) {
1272     RTC_LOG(LS_VERBOSE) << "GetSendCodec: No send codec set.";
1273     return false;
1274   }
1275   *codec = send_codec_->codec;
1276   return true;
1277 }
1278 
SetSend(bool send)1279 bool WebRtcVideoChannel::SetSend(bool send) {
1280   RTC_DCHECK_RUN_ON(&thread_checker_);
1281   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSend");
1282   RTC_LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false");
1283   if (send && !send_codec_) {
1284     RTC_DLOG(LS_ERROR) << "SetSend(true) called before setting codec.";
1285     return false;
1286   }
1287   for (const auto& kv : send_streams_) {
1288     kv.second->SetSend(send);
1289   }
1290   sending_ = send;
1291   return true;
1292 }
1293 
SetVideoSend(uint32_t ssrc,const VideoOptions * options,rtc::VideoSourceInterface<webrtc::VideoFrame> * source)1294 bool WebRtcVideoChannel::SetVideoSend(
1295     uint32_t ssrc,
1296     const VideoOptions* options,
1297     rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
1298   RTC_DCHECK_RUN_ON(&thread_checker_);
1299   TRACE_EVENT0("webrtc", "SetVideoSend");
1300   RTC_DCHECK(ssrc != 0);
1301   RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: "
1302                    << (options ? options->ToString() : "nullptr")
1303                    << ", source = " << (source ? "(source)" : "nullptr") << ")";
1304 
1305   const auto& kv = send_streams_.find(ssrc);
1306   if (kv == send_streams_.end()) {
1307     // Allow unknown ssrc only if source is null.
1308     RTC_CHECK(source == nullptr);
1309     RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc;
1310     return false;
1311   }
1312 
1313   return kv->second->SetVideoSend(options, source);
1314 }
1315 
ValidateSendSsrcAvailability(const StreamParams & sp) const1316 bool WebRtcVideoChannel::ValidateSendSsrcAvailability(
1317     const StreamParams& sp) const {
1318   for (uint32_t ssrc : sp.ssrcs) {
1319     if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) {
1320       RTC_LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc
1321                         << "' already exists.";
1322       return false;
1323     }
1324   }
1325   return true;
1326 }
1327 
ValidateReceiveSsrcAvailability(const StreamParams & sp) const1328 bool WebRtcVideoChannel::ValidateReceiveSsrcAvailability(
1329     const StreamParams& sp) const {
1330   for (uint32_t ssrc : sp.ssrcs) {
1331     if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) {
1332       RTC_LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc
1333                         << "' already exists.";
1334       return false;
1335     }
1336   }
1337   return true;
1338 }
1339 
AddSendStream(const StreamParams & sp)1340 bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) {
1341   RTC_DCHECK_RUN_ON(&thread_checker_);
1342   RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString();
1343   if (!ValidateStreamParams(sp))
1344     return false;
1345 
1346   if (!ValidateSendSsrcAvailability(sp))
1347     return false;
1348 
1349   for (uint32_t used_ssrc : sp.ssrcs)
1350     send_ssrcs_.insert(used_ssrc);
1351 
1352   webrtc::VideoSendStream::Config config(this);
1353 
1354   for (const RidDescription& rid : sp.rids()) {
1355     config.rtp.rids.push_back(rid.rid);
1356   }
1357 
1358   config.suspend_below_min_bitrate = video_config_.suspend_below_min_bitrate;
1359   config.periodic_alr_bandwidth_probing =
1360       video_config_.periodic_alr_bandwidth_probing;
1361   config.encoder_settings.experiment_cpu_load_estimator =
1362       video_config_.experiment_cpu_load_estimator;
1363   config.encoder_settings.encoder_factory = encoder_factory_;
1364   config.encoder_settings.bitrate_allocator_factory =
1365       bitrate_allocator_factory_;
1366   config.encoder_settings.encoder_switch_request_callback = this;
1367   config.crypto_options = crypto_options_;
1368   config.rtp.extmap_allow_mixed = ExtmapAllowMixed();
1369   config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms;
1370 
1371   WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
1372       call_, sp, std::move(config), default_send_options_,
1373       video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps,
1374       send_codec_, send_rtp_extensions_, send_params_);
1375 
1376   uint32_t ssrc = sp.first_ssrc();
1377   RTC_DCHECK(ssrc != 0);
1378   send_streams_[ssrc] = stream;
1379 
1380   if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
1381     SetReceiverReportSsrc(ssrc);
1382   }
1383 
1384   if (sending_) {
1385     stream->SetSend(true);
1386   }
1387 
1388   return true;
1389 }
1390 
RemoveSendStream(uint32_t ssrc)1391 bool WebRtcVideoChannel::RemoveSendStream(uint32_t ssrc) {
1392   RTC_DCHECK_RUN_ON(&thread_checker_);
1393   RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc;
1394 
1395   WebRtcVideoSendStream* removed_stream;
1396   std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
1397       send_streams_.find(ssrc);
1398   if (it == send_streams_.end()) {
1399     return false;
1400   }
1401 
1402   for (uint32_t old_ssrc : it->second->GetSsrcs())
1403     send_ssrcs_.erase(old_ssrc);
1404 
1405   removed_stream = it->second;
1406   send_streams_.erase(it);
1407 
1408   // Switch receiver report SSRCs, the one in use is no longer valid.
1409   if (rtcp_receiver_report_ssrc_ == ssrc) {
1410     SetReceiverReportSsrc(send_streams_.empty() ? kDefaultRtcpReceiverReportSsrc
1411                                                 : send_streams_.begin()->first);
1412   }
1413 
1414   delete removed_stream;
1415 
1416   return true;
1417 }
1418 
DeleteReceiveStream(WebRtcVideoChannel::WebRtcVideoReceiveStream * stream)1419 void WebRtcVideoChannel::DeleteReceiveStream(
1420     WebRtcVideoChannel::WebRtcVideoReceiveStream* stream) {
1421   for (uint32_t old_ssrc : stream->GetSsrcs())
1422     receive_ssrcs_.erase(old_ssrc);
1423   delete stream;
1424 }
1425 
AddRecvStream(const StreamParams & sp)1426 bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp) {
1427   return AddRecvStream(sp, false);
1428 }
1429 
AddRecvStream(const StreamParams & sp,bool default_stream)1430 bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp,
1431                                        bool default_stream) {
1432   RTC_DCHECK_RUN_ON(&thread_checker_);
1433 
1434   RTC_LOG(LS_INFO) << "AddRecvStream"
1435                    << (default_stream ? " (default stream)" : "") << ": "
1436                    << sp.ToString();
1437   if (!sp.has_ssrcs()) {
1438     // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used
1439     // later when we know the SSRC on the first packet arrival.
1440     unsignaled_stream_params_ = sp;
1441     return true;
1442   }
1443 
1444   if (!ValidateStreamParams(sp))
1445     return false;
1446 
1447   for (uint32_t ssrc : sp.ssrcs) {
1448     // Remove running stream if this was a default stream.
1449     const auto& prev_stream = receive_streams_.find(ssrc);
1450     if (prev_stream != receive_streams_.end()) {
1451       if (default_stream || !prev_stream->second->IsDefaultStream()) {
1452         RTC_LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc
1453                           << "' already exists.";
1454         return false;
1455       }
1456       DeleteReceiveStream(prev_stream->second);
1457       receive_streams_.erase(prev_stream);
1458     }
1459   }
1460 
1461   if (!ValidateReceiveSsrcAvailability(sp))
1462     return false;
1463 
1464   for (uint32_t used_ssrc : sp.ssrcs)
1465     receive_ssrcs_.insert(used_ssrc);
1466 
1467   webrtc::VideoReceiveStreamInterface::Config config(this, decoder_factory_);
1468   webrtc::FlexfecReceiveStream::Config flexfec_config(this);
1469   ConfigureReceiverRtp(&config, &flexfec_config, sp);
1470 
1471   config.crypto_options = crypto_options_;
1472   config.enable_prerenderer_smoothing =
1473       video_config_.enable_prerenderer_smoothing;
1474   if (!sp.stream_ids().empty()) {
1475     config.sync_group = sp.stream_ids()[0];
1476   }
1477 
1478   if (unsignaled_frame_transformer_ && !config.frame_transformer)
1479     config.frame_transformer = unsignaled_frame_transformer_;
1480 
1481   receive_streams_[sp.first_ssrc()] = new WebRtcVideoReceiveStream(
1482       this, call_, sp, std::move(config), default_stream, recv_codecs_,
1483       flexfec_config);
1484 
1485   return true;
1486 }
1487 
ConfigureReceiverRtp(webrtc::VideoReceiveStreamInterface::Config * config,webrtc::FlexfecReceiveStream::Config * flexfec_config,const StreamParams & sp) const1488 void WebRtcVideoChannel::ConfigureReceiverRtp(
1489     webrtc::VideoReceiveStreamInterface::Config* config,
1490     webrtc::FlexfecReceiveStream::Config* flexfec_config,
1491     const StreamParams& sp) const {
1492   uint32_t ssrc = sp.first_ssrc();
1493 
1494   config->rtp.remote_ssrc = ssrc;
1495   config->rtp.local_ssrc = rtcp_receiver_report_ssrc_;
1496 
1497   // TODO(pbos): This protection is against setting the same local ssrc as
1498   // remote which is not permitted by the lower-level API. RTCP requires a
1499   // corresponding sender SSRC. Figure out what to do when we don't have
1500   // (receive-only) or know a good local SSRC.
1501   if (config->rtp.remote_ssrc == config->rtp.local_ssrc) {
1502     if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) {
1503       config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc;
1504     } else {
1505       config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1;
1506     }
1507   }
1508 
1509   // Whether or not the receive stream sends reduced size RTCP is determined
1510   // by the send params.
1511   // TODO(deadbeef): Once we change "send_params" to "sender_params" and
1512   // "recv_params" to "receiver_params", we should get this out of
1513   // receiver_params_.
1514   config->rtp.rtcp_mode = send_params_.rtcp.reduced_size
1515                               ? webrtc::RtcpMode::kReducedSize
1516                               : webrtc::RtcpMode::kCompound;
1517 
1518   // rtx-time (RFC 4588) is a declarative attribute similar to rtcp-rsize and
1519   // determined by the sender / send codec.
1520   if (send_codec_ && send_codec_->rtx_time != -1) {
1521     config->rtp.nack.rtp_history_ms = send_codec_->rtx_time;
1522   }
1523 
1524   config->rtp.transport_cc =
1525       send_codec_ ? HasTransportCc(send_codec_->codec) : false;
1526 
1527   sp.GetFidSsrc(ssrc, &config->rtp.rtx_ssrc);
1528 
1529   config->rtp.extensions = recv_rtp_extensions_;
1530 
1531   // TODO(brandtr): Generalize when we add support for multistream protection.
1532   flexfec_config->payload_type = recv_flexfec_payload_type_;
1533   if (!IsDisabled(call_->trials(), "WebRTC-FlexFEC-03-Advertised") &&
1534       sp.GetFecFrSsrc(ssrc, &flexfec_config->rtp.remote_ssrc)) {
1535     flexfec_config->protected_media_ssrcs = {ssrc};
1536     flexfec_config->rtp.local_ssrc = config->rtp.local_ssrc;
1537     flexfec_config->rtcp_mode = config->rtp.rtcp_mode;
1538     // TODO(brandtr): We should be spec-compliant and set `transport_cc` here
1539     // based on the rtcp-fb for the FlexFEC codec, not the media codec.
1540     flexfec_config->rtp.transport_cc = config->rtp.transport_cc;
1541     flexfec_config->rtp.extensions = config->rtp.extensions;
1542   }
1543 }
1544 
RemoveRecvStream(uint32_t ssrc)1545 bool WebRtcVideoChannel::RemoveRecvStream(uint32_t ssrc) {
1546   RTC_DCHECK_RUN_ON(&thread_checker_);
1547   RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc;
1548 
1549   std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream =
1550       receive_streams_.find(ssrc);
1551   if (stream == receive_streams_.end()) {
1552     RTC_LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc;
1553     return false;
1554   }
1555   DeleteReceiveStream(stream->second);
1556   receive_streams_.erase(stream);
1557 
1558   return true;
1559 }
1560 
ResetUnsignaledRecvStream()1561 void WebRtcVideoChannel::ResetUnsignaledRecvStream() {
1562   RTC_DCHECK_RUN_ON(&thread_checker_);
1563   RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream.";
1564   unsignaled_stream_params_ = StreamParams();
1565   last_unsignalled_ssrc_creation_time_ms_ = absl::nullopt;
1566 
1567   // Delete any created default streams. This is needed to avoid SSRC collisions
1568   // in Call's RtpDemuxer, in the case that `this` has created a default video
1569   // receiver, and then some other WebRtcVideoChannel gets the SSRC signaled
1570   // in the corresponding Unified Plan "m=" section.
1571   auto it = receive_streams_.begin();
1572   while (it != receive_streams_.end()) {
1573     if (it->second->IsDefaultStream()) {
1574       DeleteReceiveStream(it->second);
1575       receive_streams_.erase(it++);
1576     } else {
1577       ++it;
1578     }
1579   }
1580 }
1581 
OnDemuxerCriteriaUpdatePending()1582 void WebRtcVideoChannel::OnDemuxerCriteriaUpdatePending() {
1583   RTC_DCHECK_RUN_ON(&thread_checker_);
1584   ++demuxer_criteria_id_;
1585 }
1586 
OnDemuxerCriteriaUpdateComplete()1587 void WebRtcVideoChannel::OnDemuxerCriteriaUpdateComplete() {
1588   RTC_DCHECK_RUN_ON(&thread_checker_);
1589   ++demuxer_criteria_completed_id_;
1590 }
1591 
SetSink(uint32_t ssrc,rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)1592 bool WebRtcVideoChannel::SetSink(
1593     uint32_t ssrc,
1594     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
1595   RTC_DCHECK_RUN_ON(&thread_checker_);
1596   RTC_LOG(LS_INFO) << "SetSink: ssrc:" << ssrc << " "
1597                    << (sink ? "(ptr)" : "nullptr");
1598 
1599   std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
1600       receive_streams_.find(ssrc);
1601   if (it == receive_streams_.end()) {
1602     return false;
1603   }
1604 
1605   it->second->SetSink(sink);
1606   return true;
1607 }
1608 
SetDefaultSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)1609 void WebRtcVideoChannel::SetDefaultSink(
1610     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
1611   RTC_DCHECK_RUN_ON(&thread_checker_);
1612   RTC_LOG(LS_INFO) << "SetDefaultSink: " << (sink ? "(ptr)" : "nullptr");
1613   default_unsignalled_ssrc_handler_.SetDefaultSink(this, sink);
1614 }
1615 
GetStats(VideoMediaInfo * info)1616 bool WebRtcVideoChannel::GetStats(VideoMediaInfo* info) {
1617   RTC_DCHECK_RUN_ON(&thread_checker_);
1618   TRACE_EVENT0("webrtc", "WebRtcVideoChannel::GetStats");
1619 
1620   // Log stats periodically.
1621   bool log_stats = false;
1622   int64_t now_ms = rtc::TimeMillis();
1623   if (last_stats_log_ms_ == -1 ||
1624       now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
1625     last_stats_log_ms_ = now_ms;
1626     log_stats = true;
1627   }
1628 
1629   info->Clear();
1630   FillSenderStats(info, log_stats);
1631   FillReceiverStats(info, log_stats);
1632   FillSendAndReceiveCodecStats(info);
1633   // TODO(holmer): We should either have rtt available as a metric on
1634   // VideoSend/ReceiveStreams, or we should remove rtt from VideoSenderInfo.
1635   webrtc::Call::Stats stats = call_->GetStats();
1636   if (stats.rtt_ms != -1) {
1637     for (size_t i = 0; i < info->senders.size(); ++i) {
1638       info->senders[i].rtt_ms = stats.rtt_ms;
1639     }
1640     for (size_t i = 0; i < info->aggregated_senders.size(); ++i) {
1641       info->aggregated_senders[i].rtt_ms = stats.rtt_ms;
1642     }
1643   }
1644 
1645   if (log_stats)
1646     RTC_LOG(LS_INFO) << stats.ToString(now_ms);
1647 
1648   return true;
1649 }
1650 
FillSenderStats(VideoMediaInfo * video_media_info,bool log_stats)1651 void WebRtcVideoChannel::FillSenderStats(VideoMediaInfo* video_media_info,
1652                                          bool log_stats) {
1653   for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
1654            send_streams_.begin();
1655        it != send_streams_.end(); ++it) {
1656     auto infos = it->second->GetPerLayerVideoSenderInfos(log_stats);
1657     if (infos.empty())
1658       continue;
1659     video_media_info->aggregated_senders.push_back(
1660         it->second->GetAggregatedVideoSenderInfo(infos));
1661     for (auto&& info : infos) {
1662       video_media_info->senders.push_back(info);
1663     }
1664   }
1665 }
1666 
FillReceiverStats(VideoMediaInfo * video_media_info,bool log_stats)1667 void WebRtcVideoChannel::FillReceiverStats(VideoMediaInfo* video_media_info,
1668                                            bool log_stats) {
1669   for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
1670            receive_streams_.begin();
1671        it != receive_streams_.end(); ++it) {
1672     video_media_info->receivers.push_back(
1673         it->second->GetVideoReceiverInfo(log_stats));
1674   }
1675 }
1676 
FillBitrateInfo(BandwidthEstimationInfo * bwe_info)1677 void WebRtcVideoChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) {
1678   RTC_DCHECK_RUN_ON(&thread_checker_);
1679   for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream =
1680            send_streams_.begin();
1681        stream != send_streams_.end(); ++stream) {
1682     stream->second->FillBitrateInfo(bwe_info);
1683   }
1684 }
1685 
FillSendAndReceiveCodecStats(VideoMediaInfo * video_media_info)1686 void WebRtcVideoChannel::FillSendAndReceiveCodecStats(
1687     VideoMediaInfo* video_media_info) {
1688   for (const VideoCodec& codec : send_params_.codecs) {
1689     webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
1690     video_media_info->send_codecs.insert(
1691         std::make_pair(codec_params.payload_type, std::move(codec_params)));
1692   }
1693   for (const VideoCodec& codec : recv_params_.codecs) {
1694     webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
1695     video_media_info->receive_codecs.insert(
1696         std::make_pair(codec_params.payload_type, std::move(codec_params)));
1697   }
1698 }
1699 
OnPacketReceived(rtc::CopyOnWriteBuffer packet,int64_t packet_time_us)1700 void WebRtcVideoChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet,
1701                                           int64_t packet_time_us) {
1702   RTC_DCHECK_RUN_ON(&network_thread_checker_);
1703   // TODO(bugs.webrtc.org/11993): This code is very similar to what
1704   // WebRtcVoiceMediaChannel::OnPacketReceived does. For maintainability and
1705   // consistency it would be good to move the interaction with call_->Receiver()
1706   // to a common implementation and provide a callback on the worker thread
1707   // for the exception case (DELIVERY_UNKNOWN_SSRC) and how retry is attempted.
1708   worker_thread_->PostTask(
1709       SafeTask(task_safety_.flag(), [this, packet, packet_time_us] {
1710         RTC_DCHECK_RUN_ON(&thread_checker_);
1711         const webrtc::PacketReceiver::DeliveryStatus delivery_result =
1712             call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1713                                              packet_time_us);
1714         switch (delivery_result) {
1715           case webrtc::PacketReceiver::DELIVERY_OK:
1716             return;
1717           case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
1718             return;
1719           case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
1720             break;
1721         }
1722 
1723         absl::optional<uint32_t> rtx_ssrc;
1724         uint32_t ssrc = ParseRtpSsrc(packet);
1725 
1726         if (unknown_ssrc_packet_buffer_) {
1727           unknown_ssrc_packet_buffer_->AddPacket(ssrc, packet_time_us, packet);
1728           return;
1729         }
1730 
1731         if (discard_unknown_ssrc_packets_) {
1732           return;
1733         }
1734 
1735         int payload_type = ParseRtpPayloadType(packet);
1736 
1737         // See if this payload_type is registered as one that usually gets its
1738         // own SSRC (RTX) or at least is safe to drop either way (FEC). If it
1739         // is, and it wasn't handled above by DeliverPacket, that means we don't
1740         // know what stream it associates with, and we shouldn't ever create an
1741         // implicit channel for these.
1742         for (auto& codec : recv_codecs_) {
1743           if (payload_type == codec.ulpfec.red_rtx_payload_type ||
1744               payload_type == codec.ulpfec.ulpfec_payload_type) {
1745             return;
1746           }
1747           if (payload_type == codec.rtx_payload_type) {
1748             // As we don't support receiving simulcast there can only be one RTX
1749             // stream, which will be associated with unsignaled media stream.
1750             // It is not possible to update the ssrcs of a receive stream, so we
1751             // recreate it insead if found.
1752             auto default_ssrc = GetDefaultReceiveStreamSsrc();
1753             if (!default_ssrc) {
1754               return;
1755             }
1756             rtx_ssrc = ssrc;
1757             ssrc = *default_ssrc;
1758             // Allow recreating the receive stream even if the RTX packet is
1759             // received just after the media packet.
1760             last_unsignalled_ssrc_creation_time_ms_.reset();
1761             break;
1762           }
1763         }
1764         if (payload_type == recv_flexfec_payload_type_) {
1765           return;
1766         }
1767 
1768         // Ignore unknown ssrcs if there is a demuxer criteria update pending.
1769         // During a demuxer update we may receive ssrcs that were recently
1770         // removed or we may receve ssrcs that were recently configured for a
1771         // different video channel.
1772         if (demuxer_criteria_id_ != demuxer_criteria_completed_id_) {
1773           return;
1774         }
1775         // Ignore unknown ssrcs if we recently created an unsignalled receive
1776         // stream since this shouldn't happen frequently. Getting into a state
1777         // of creating decoders on every packet eats up processing time (e.g.
1778         // https://crbug.com/1069603) and this cooldown prevents that.
1779         if (last_unsignalled_ssrc_creation_time_ms_.has_value()) {
1780           int64_t now_ms = rtc::TimeMillis();
1781           if (now_ms - last_unsignalled_ssrc_creation_time_ms_.value() <
1782               kUnsignaledSsrcCooldownMs) {
1783             // We've already created an unsignalled ssrc stream within the last
1784             // 0.5 s, ignore with a warning.
1785             RTC_LOG(LS_WARNING)
1786                 << "Another unsignalled ssrc packet arrived shortly after the "
1787                 << "creation of an unsignalled ssrc stream. Dropping packet.";
1788             return;
1789           }
1790         }
1791         // Let the unsignalled ssrc handler decide whether to drop or deliver.
1792         switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc,
1793                                                              rtx_ssrc)) {
1794           case UnsignalledSsrcHandler::kDropPacket:
1795             return;
1796           case UnsignalledSsrcHandler::kDeliverPacket:
1797             break;
1798         }
1799 
1800         if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1801                                              packet_time_us) !=
1802             webrtc::PacketReceiver::DELIVERY_OK) {
1803           RTC_LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery.";
1804         }
1805         last_unsignalled_ssrc_creation_time_ms_ = rtc::TimeMillis();
1806       }));
1807 }
1808 
OnPacketSent(const rtc::SentPacket & sent_packet)1809 void WebRtcVideoChannel::OnPacketSent(const rtc::SentPacket& sent_packet) {
1810   RTC_DCHECK_RUN_ON(&network_thread_checker_);
1811   // TODO(tommi): We shouldn't need to go through call_ to deliver this
1812   // notification. We should already have direct access to
1813   // video_send_delay_stats_ and transport_send_ptr_ via `stream_`.
1814   // So we should be able to remove OnSentPacket from Call and handle this per
1815   // channel instead. At the moment Call::OnSentPacket calls OnSentPacket for
1816   // the video stats, for all sent packets, including audio, which causes
1817   // unnecessary lookups.
1818   call_->OnSentPacket(sent_packet);
1819 }
1820 
BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs)1821 void WebRtcVideoChannel::BackfillBufferedPackets(
1822     rtc::ArrayView<const uint32_t> ssrcs) {
1823   RTC_DCHECK_RUN_ON(&thread_checker_);
1824   if (!unknown_ssrc_packet_buffer_) {
1825     return;
1826   }
1827 
1828   int delivery_ok_cnt = 0;
1829   int delivery_unknown_ssrc_cnt = 0;
1830   int delivery_packet_error_cnt = 0;
1831   webrtc::PacketReceiver* receiver = this->call_->Receiver();
1832   unknown_ssrc_packet_buffer_->BackfillPackets(
1833       ssrcs, [&](uint32_t /*ssrc*/, int64_t packet_time_us,
1834                  rtc::CopyOnWriteBuffer packet) {
1835         switch (receiver->DeliverPacket(webrtc::MediaType::VIDEO, packet,
1836                                         packet_time_us)) {
1837           case webrtc::PacketReceiver::DELIVERY_OK:
1838             delivery_ok_cnt++;
1839             break;
1840           case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
1841             delivery_unknown_ssrc_cnt++;
1842             break;
1843           case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
1844             delivery_packet_error_cnt++;
1845             break;
1846         }
1847       });
1848   rtc::StringBuilder out;
1849   out << "[ ";
1850   for (uint32_t ssrc : ssrcs) {
1851     out << std::to_string(ssrc) << " ";
1852   }
1853   out << "]";
1854   auto level = rtc::LS_INFO;
1855   if (delivery_unknown_ssrc_cnt > 0 || delivery_packet_error_cnt > 0) {
1856     level = rtc::LS_ERROR;
1857   }
1858   int total =
1859       delivery_ok_cnt + delivery_unknown_ssrc_cnt + delivery_packet_error_cnt;
1860   RTC_LOG_V(level) << "Backfilled " << total
1861                    << " packets for ssrcs: " << out.Release()
1862                    << " ok: " << delivery_ok_cnt
1863                    << " error: " << delivery_packet_error_cnt
1864                    << " unknown: " << delivery_unknown_ssrc_cnt;
1865 }
1866 
OnReadyToSend(bool ready)1867 void WebRtcVideoChannel::OnReadyToSend(bool ready) {
1868   RTC_DCHECK_RUN_ON(&network_thread_checker_);
1869   RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready.");
1870   call_->SignalChannelNetworkState(
1871       webrtc::MediaType::VIDEO,
1872       ready ? webrtc::kNetworkUp : webrtc::kNetworkDown);
1873 }
1874 
OnNetworkRouteChanged(absl::string_view transport_name,const rtc::NetworkRoute & network_route)1875 void WebRtcVideoChannel::OnNetworkRouteChanged(
1876     absl::string_view transport_name,
1877     const rtc::NetworkRoute& network_route) {
1878   RTC_DCHECK_RUN_ON(&network_thread_checker_);
1879   worker_thread_->PostTask(SafeTask(
1880       task_safety_.flag(),
1881       [this, name = std::string(transport_name), route = network_route] {
1882         RTC_DCHECK_RUN_ON(&thread_checker_);
1883         webrtc::RtpTransportControllerSendInterface* transport =
1884             call_->GetTransportControllerSend();
1885         transport->OnNetworkRouteChanged(name, route);
1886         transport->OnTransportOverheadChanged(route.packet_overhead);
1887       }));
1888 }
1889 
SetInterface(NetworkInterface * iface)1890 void WebRtcVideoChannel::SetInterface(NetworkInterface* iface) {
1891   RTC_DCHECK_RUN_ON(&network_thread_checker_);
1892   MediaChannel::SetInterface(iface);
1893   // Set the RTP recv/send buffer to a bigger size.
1894   MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF,
1895                           kVideoRtpRecvBufferSize);
1896 
1897   // Speculative change to increase the outbound socket buffer size.
1898   // In b/15152257, we are seeing a significant number of packets discarded
1899   // due to lack of socket buffer space, although it's not yet clear what the
1900   // ideal value should be.
1901   const std::string group_name_send_buf_size =
1902       call_->trials().Lookup("WebRTC-SendBufferSizeBytes");
1903   int send_buffer_size = kVideoRtpSendBufferSize;
1904   if (!group_name_send_buf_size.empty() &&
1905       (sscanf(group_name_send_buf_size.c_str(), "%d", &send_buffer_size) != 1 ||
1906        send_buffer_size <= 0)) {
1907     RTC_LOG(LS_WARNING) << "Invalid send buffer size: "
1908                         << group_name_send_buf_size;
1909     send_buffer_size = kVideoRtpSendBufferSize;
1910   }
1911 
1912   MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF,
1913                           send_buffer_size);
1914 }
1915 
SetFrameDecryptor(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)1916 void WebRtcVideoChannel::SetFrameDecryptor(
1917     uint32_t ssrc,
1918     rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
1919   RTC_DCHECK_RUN_ON(&thread_checker_);
1920   auto matching_stream = receive_streams_.find(ssrc);
1921   if (matching_stream != receive_streams_.end()) {
1922     matching_stream->second->SetFrameDecryptor(frame_decryptor);
1923   }
1924 }
1925 
SetFrameEncryptor(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor)1926 void WebRtcVideoChannel::SetFrameEncryptor(
1927     uint32_t ssrc,
1928     rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
1929   RTC_DCHECK_RUN_ON(&thread_checker_);
1930   auto matching_stream = send_streams_.find(ssrc);
1931   if (matching_stream != send_streams_.end()) {
1932     matching_stream->second->SetFrameEncryptor(frame_encryptor);
1933   } else {
1934     RTC_LOG(LS_ERROR) << "No stream found to attach frame encryptor";
1935   }
1936 }
1937 
SetEncoderSelector(uint32_t ssrc,webrtc::VideoEncoderFactory::EncoderSelectorInterface * encoder_selector)1938 void WebRtcVideoChannel::SetEncoderSelector(
1939     uint32_t ssrc,
1940     webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
1941   RTC_DCHECK_RUN_ON(&thread_checker_);
1942   auto matching_stream = send_streams_.find(ssrc);
1943   if (matching_stream != send_streams_.end()) {
1944     matching_stream->second->SetEncoderSelector(encoder_selector);
1945   } else {
1946     RTC_LOG(LS_ERROR) << "No stream found to attach encoder selector";
1947   }
1948 }
1949 
SetVideoCodecSwitchingEnabled(bool enabled)1950 void WebRtcVideoChannel::SetVideoCodecSwitchingEnabled(bool enabled) {
1951   RTC_DCHECK_RUN_ON(&thread_checker_);
1952   allow_codec_switching_ = enabled;
1953   if (allow_codec_switching_) {
1954     RTC_LOG(LS_INFO) << "Encoder switching enabled.";
1955   }
1956 }
1957 
SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,int delay_ms)1958 bool WebRtcVideoChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,
1959                                                       int delay_ms) {
1960   RTC_DCHECK_RUN_ON(&thread_checker_);
1961   absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
1962 
1963   // SSRC of 0 represents the default receive stream.
1964   if (ssrc == 0) {
1965     default_recv_base_minimum_delay_ms_ = delay_ms;
1966   }
1967 
1968   if (ssrc == 0 && !default_ssrc) {
1969     return true;
1970   }
1971 
1972   if (ssrc == 0 && default_ssrc) {
1973     ssrc = default_ssrc.value();
1974   }
1975 
1976   auto stream = receive_streams_.find(ssrc);
1977   if (stream != receive_streams_.end()) {
1978     stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms);
1979     return true;
1980   } else {
1981     RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay";
1982     return false;
1983   }
1984 }
1985 
GetBaseMinimumPlayoutDelayMs(uint32_t ssrc) const1986 absl::optional<int> WebRtcVideoChannel::GetBaseMinimumPlayoutDelayMs(
1987     uint32_t ssrc) const {
1988   RTC_DCHECK_RUN_ON(&thread_checker_);
1989   // SSRC of 0 represents the default receive stream.
1990   if (ssrc == 0) {
1991     return default_recv_base_minimum_delay_ms_;
1992   }
1993 
1994   auto stream = receive_streams_.find(ssrc);
1995   if (stream != receive_streams_.end()) {
1996     return stream->second->GetBaseMinimumPlayoutDelayMs();
1997   } else {
1998     RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay";
1999     return absl::nullopt;
2000   }
2001 }
2002 
GetDefaultReceiveStreamSsrc()2003 absl::optional<uint32_t> WebRtcVideoChannel::GetDefaultReceiveStreamSsrc() {
2004   RTC_DCHECK_RUN_ON(&thread_checker_);
2005   absl::optional<uint32_t> ssrc;
2006   for (auto it = receive_streams_.begin(); it != receive_streams_.end(); ++it) {
2007     if (it->second->IsDefaultStream()) {
2008       ssrc.emplace(it->first);
2009       break;
2010     }
2011   }
2012   return ssrc;
2013 }
2014 
GetSources(uint32_t ssrc) const2015 std::vector<webrtc::RtpSource> WebRtcVideoChannel::GetSources(
2016     uint32_t ssrc) const {
2017   RTC_DCHECK_RUN_ON(&thread_checker_);
2018   auto it = receive_streams_.find(ssrc);
2019   if (it == receive_streams_.end()) {
2020     // TODO(bugs.webrtc.org/9781): Investigate standard compliance
2021     // with sources for streams that has been removed.
2022     RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:"
2023                       << ssrc << " which doesn't exist.";
2024     return {};
2025   }
2026   return it->second->GetSources();
2027 }
2028 
SendRtp(const uint8_t * data,size_t len,const webrtc::PacketOptions & options)2029 bool WebRtcVideoChannel::SendRtp(const uint8_t* data,
2030                                  size_t len,
2031                                  const webrtc::PacketOptions& options) {
2032   MediaChannel::SendRtp(data, len, options);
2033   return true;
2034 }
2035 
SendRtcp(const uint8_t * data,size_t len)2036 bool WebRtcVideoChannel::SendRtcp(const uint8_t* data, size_t len) {
2037   MediaChannel::SendRtcp(data, len);
2038   return true;
2039 }
2040 
2041 WebRtcVideoChannel::WebRtcVideoSendStream::VideoSendStreamParameters::
VideoSendStreamParameters(webrtc::VideoSendStream::Config config,const VideoOptions & options,int max_bitrate_bps,const absl::optional<VideoCodecSettings> & codec_settings)2042     VideoSendStreamParameters(
2043         webrtc::VideoSendStream::Config config,
2044         const VideoOptions& options,
2045         int max_bitrate_bps,
2046         const absl::optional<VideoCodecSettings>& codec_settings)
2047     : config(std::move(config)),
2048       options(options),
2049       max_bitrate_bps(max_bitrate_bps),
2050       conference_mode(false),
2051       codec_settings(codec_settings) {}
2052 
WebRtcVideoSendStream(webrtc::Call * call,const StreamParams & sp,webrtc::VideoSendStream::Config config,const VideoOptions & options,bool enable_cpu_overuse_detection,int max_bitrate_bps,const absl::optional<VideoCodecSettings> & codec_settings,const absl::optional<std::vector<webrtc::RtpExtension>> & rtp_extensions,const VideoSendParameters & send_params)2053 WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
2054     webrtc::Call* call,
2055     const StreamParams& sp,
2056     webrtc::VideoSendStream::Config config,
2057     const VideoOptions& options,
2058     bool enable_cpu_overuse_detection,
2059     int max_bitrate_bps,
2060     const absl::optional<VideoCodecSettings>& codec_settings,
2061     const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
2062     // TODO(deadbeef): Don't duplicate information between send_params,
2063     // rtp_extensions, options, etc.
2064     const VideoSendParameters& send_params)
2065     : worker_thread_(call->worker_thread()),
2066       ssrcs_(sp.ssrcs),
2067       ssrc_groups_(sp.ssrc_groups),
2068       call_(call),
2069       enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
2070       source_(nullptr),
2071       stream_(nullptr),
2072       parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
2073       rtp_parameters_(CreateRtpParametersWithEncodings(sp)),
2074       sending_(false),
2075       disable_automatic_resize_(
2076           IsEnabled(call->trials(), "WebRTC-Video-DisableAutomaticResize")) {
2077   // Maximum packet size may come in RtpConfig from external transport, for
2078   // example from QuicTransportInterface implementation, so do not exceed
2079   // given max_packet_size.
2080   parameters_.config.rtp.max_packet_size =
2081       std::min<size_t>(parameters_.config.rtp.max_packet_size, kVideoMtu);
2082   parameters_.conference_mode = send_params.conference_mode;
2083 
2084   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
2085 
2086   // ValidateStreamParams should prevent this from happening.
2087   RTC_CHECK(!parameters_.config.rtp.ssrcs.empty());
2088   rtp_parameters_.encodings[0].ssrc = parameters_.config.rtp.ssrcs[0];
2089 
2090   // RTX.
2091   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
2092                  &parameters_.config.rtp.rtx.ssrcs);
2093 
2094   // FlexFEC SSRCs.
2095   // TODO(brandtr): This code needs to be generalized when we add support for
2096   // multistream protection.
2097   if (IsEnabled(call_->trials(), "WebRTC-FlexFEC-03")) {
2098     uint32_t flexfec_ssrc;
2099     bool flexfec_enabled = false;
2100     for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) {
2101       if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) {
2102         if (flexfec_enabled) {
2103           RTC_LOG(LS_INFO)
2104               << "Multiple FlexFEC streams in local SDP, but "
2105                  "our implementation only supports a single FlexFEC "
2106                  "stream. Will not enable FlexFEC for proposed "
2107                  "stream with SSRC: "
2108               << flexfec_ssrc << ".";
2109           continue;
2110         }
2111 
2112         flexfec_enabled = true;
2113         parameters_.config.rtp.flexfec.ssrc = flexfec_ssrc;
2114         parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc};
2115       }
2116     }
2117   }
2118 
2119   parameters_.config.rtp.c_name = sp.cname;
2120   if (rtp_extensions) {
2121     parameters_.config.rtp.extensions = *rtp_extensions;
2122     rtp_parameters_.header_extensions = *rtp_extensions;
2123   }
2124   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
2125                                          ? webrtc::RtcpMode::kReducedSize
2126                                          : webrtc::RtcpMode::kCompound;
2127   parameters_.config.rtp.mid = send_params.mid;
2128   rtp_parameters_.rtcp.reduced_size = send_params.rtcp.reduced_size;
2129 
2130   if (codec_settings) {
2131     SetCodec(*codec_settings);
2132   }
2133 }
2134 
~WebRtcVideoSendStream()2135 WebRtcVideoChannel::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
2136   if (stream_ != NULL) {
2137     call_->DestroyVideoSendStream(stream_);
2138   }
2139 }
2140 
SetVideoSend(const VideoOptions * options,rtc::VideoSourceInterface<webrtc::VideoFrame> * source)2141 bool WebRtcVideoChannel::WebRtcVideoSendStream::SetVideoSend(
2142     const VideoOptions* options,
2143     rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
2144   TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend");
2145   RTC_DCHECK_RUN_ON(&thread_checker_);
2146 
2147   if (options) {
2148     VideoOptions old_options = parameters_.options;
2149     parameters_.options.SetAll(*options);
2150     if (parameters_.options.is_screencast.value_or(false) !=
2151             old_options.is_screencast.value_or(false) &&
2152         parameters_.codec_settings) {
2153       // If screen content settings change, we may need to recreate the codec
2154       // instance so that the correct type is used.
2155 
2156       SetCodec(*parameters_.codec_settings);
2157       // Mark screenshare parameter as being updated, then test for any other
2158       // changes that may require codec reconfiguration.
2159       old_options.is_screencast = options->is_screencast;
2160     }
2161     if (parameters_.options != old_options) {
2162       ReconfigureEncoder(nullptr);
2163     }
2164   }
2165 
2166   if (source_ && stream_) {
2167     stream_->SetSource(nullptr, webrtc::DegradationPreference::DISABLED);
2168   }
2169   // Switch to the new source.
2170   source_ = source;
2171   if (source && stream_) {
2172     stream_->SetSource(source_, GetDegradationPreference());
2173   }
2174   return true;
2175 }
2176 
2177 webrtc::DegradationPreference
GetDegradationPreference() const2178 WebRtcVideoChannel::WebRtcVideoSendStream::GetDegradationPreference() const {
2179   // Do not adapt resolution for screen content as this will likely
2180   // result in blurry and unreadable text.
2181   // `this` acts like a VideoSource to make sure SinkWants are handled on the
2182   // correct thread.
2183   if (!enable_cpu_overuse_detection_) {
2184     return webrtc::DegradationPreference::DISABLED;
2185   }
2186 
2187   webrtc::DegradationPreference degradation_preference;
2188   if (rtp_parameters_.degradation_preference.has_value()) {
2189     degradation_preference = *rtp_parameters_.degradation_preference;
2190   } else {
2191     if (parameters_.options.content_hint ==
2192         webrtc::VideoTrackInterface::ContentHint::kFluid) {
2193       degradation_preference =
2194           webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
2195     } else if (parameters_.options.is_screencast.value_or(false) ||
2196                parameters_.options.content_hint ==
2197                    webrtc::VideoTrackInterface::ContentHint::kDetailed ||
2198                parameters_.options.content_hint ==
2199                    webrtc::VideoTrackInterface::ContentHint::kText) {
2200       degradation_preference =
2201           webrtc::DegradationPreference::MAINTAIN_RESOLUTION;
2202     } else if (IsEnabled(call_->trials(), "WebRTC-Video-BalancedDegradation")) {
2203       // Standard wants balanced by default, but it needs to be tuned first.
2204       degradation_preference = webrtc::DegradationPreference::BALANCED;
2205     } else {
2206       // Keep MAINTAIN_FRAMERATE by default until BALANCED has been tuned for
2207       // all codecs and launched.
2208       degradation_preference =
2209           webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
2210     }
2211   }
2212 
2213   return degradation_preference;
2214 }
2215 
2216 const std::vector<uint32_t>&
GetSsrcs() const2217 WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
2218   return ssrcs_;
2219 }
2220 
SetCodec(const VideoCodecSettings & codec_settings)2221 void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
2222     const VideoCodecSettings& codec_settings) {
2223   RTC_DCHECK_RUN_ON(&thread_checker_);
2224   parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
2225   RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
2226 
2227   parameters_.config.rtp.payload_name = codec_settings.codec.name;
2228   parameters_.config.rtp.payload_type = codec_settings.codec.id;
2229   parameters_.config.rtp.raw_payload =
2230       codec_settings.codec.packetization == kPacketizationParamRaw;
2231   parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
2232   parameters_.config.rtp.flexfec.payload_type =
2233       codec_settings.flexfec_payload_type;
2234 
2235   // Set RTX payload type if RTX is enabled.
2236   if (!parameters_.config.rtp.rtx.ssrcs.empty()) {
2237     if (codec_settings.rtx_payload_type == -1) {
2238       RTC_LOG(LS_WARNING)
2239           << "RTX SSRCs configured but there's no configured RTX "
2240              "payload type. Ignoring.";
2241       parameters_.config.rtp.rtx.ssrcs.clear();
2242     } else {
2243       parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type;
2244     }
2245   }
2246 
2247   const bool has_lntf = HasLntf(codec_settings.codec);
2248   parameters_.config.rtp.lntf.enabled = has_lntf;
2249   parameters_.config.encoder_settings.capabilities.loss_notification = has_lntf;
2250 
2251   parameters_.config.rtp.nack.rtp_history_ms =
2252       HasNack(codec_settings.codec) ? kNackHistoryMs : 0;
2253 
2254   parameters_.codec_settings = codec_settings;
2255 
2256   // TODO(bugs.webrtc.org/8830): Avoid recreation, it should be enough to call
2257   // ReconfigureEncoder.
2258   RTC_LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetCodec.";
2259   RecreateWebRtcStream();
2260 }
2261 
SetSendParameters(const ChangedSendParameters & params)2262 void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters(
2263     const ChangedSendParameters& params) {
2264   RTC_DCHECK_RUN_ON(&thread_checker_);
2265   // `recreate_stream` means construction-time parameters have changed and the
2266   // sending stream needs to be reset with the new config.
2267   bool recreate_stream = false;
2268   if (params.rtcp_mode) {
2269     parameters_.config.rtp.rtcp_mode = *params.rtcp_mode;
2270     rtp_parameters_.rtcp.reduced_size =
2271         parameters_.config.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
2272     recreate_stream = true;
2273   }
2274   if (params.extmap_allow_mixed) {
2275     parameters_.config.rtp.extmap_allow_mixed = *params.extmap_allow_mixed;
2276     recreate_stream = true;
2277   }
2278   if (params.rtp_header_extensions) {
2279     parameters_.config.rtp.extensions = *params.rtp_header_extensions;
2280     rtp_parameters_.header_extensions = *params.rtp_header_extensions;
2281     recreate_stream = true;
2282   }
2283   if (params.mid) {
2284     parameters_.config.rtp.mid = *params.mid;
2285     recreate_stream = true;
2286   }
2287   if (params.max_bandwidth_bps) {
2288     parameters_.max_bitrate_bps = *params.max_bandwidth_bps;
2289     ReconfigureEncoder(nullptr);
2290   }
2291   if (params.conference_mode) {
2292     parameters_.conference_mode = *params.conference_mode;
2293   }
2294 
2295   // Set codecs and options.
2296   if (params.send_codec) {
2297     SetCodec(*params.send_codec);
2298     recreate_stream = false;  // SetCodec has already recreated the stream.
2299   } else if (params.conference_mode && parameters_.codec_settings) {
2300     SetCodec(*parameters_.codec_settings);
2301     recreate_stream = false;  // SetCodec has already recreated the stream.
2302   }
2303   if (recreate_stream) {
2304     RTC_LOG(LS_INFO)
2305         << "RecreateWebRtcStream (send) because of SetSendParameters";
2306     RecreateWebRtcStream();
2307   }
2308 }
2309 
SetRtpParameters(const webrtc::RtpParameters & new_parameters,webrtc::SetParametersCallback callback)2310 webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
2311     const webrtc::RtpParameters& new_parameters,
2312     webrtc::SetParametersCallback callback) {
2313   RTC_DCHECK_RUN_ON(&thread_checker_);
2314   // This is checked higher in the stack (RtpSender), so this is only checking
2315   // for users accessing the private APIs or tests, not specification
2316   // conformance.
2317   // TODO(orphis): Migrate tests to later make this a DCHECK only
2318   webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues(
2319       rtp_parameters_, new_parameters);
2320   if (!error.ok()) {
2321     // Error is propagated to the callback at a higher level
2322     return error;
2323   }
2324 
2325   bool new_param = false;
2326   for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
2327     if ((new_parameters.encodings[i].min_bitrate_bps !=
2328          rtp_parameters_.encodings[i].min_bitrate_bps) ||
2329         (new_parameters.encodings[i].max_bitrate_bps !=
2330          rtp_parameters_.encodings[i].max_bitrate_bps) ||
2331         (new_parameters.encodings[i].max_framerate !=
2332          rtp_parameters_.encodings[i].max_framerate) ||
2333         (new_parameters.encodings[i].scale_resolution_down_by !=
2334          rtp_parameters_.encodings[i].scale_resolution_down_by) ||
2335         (new_parameters.encodings[i].num_temporal_layers !=
2336          rtp_parameters_.encodings[i].num_temporal_layers) ||
2337         (new_parameters.encodings[i].requested_resolution !=
2338          rtp_parameters_.encodings[i].requested_resolution)) {
2339       new_param = true;
2340       break;
2341     }
2342   }
2343 
2344   bool new_degradation_preference = false;
2345   if (new_parameters.degradation_preference !=
2346       rtp_parameters_.degradation_preference) {
2347     new_degradation_preference = true;
2348   }
2349 
2350   // Some fields (e.g. bitrate priority) only need to update the bitrate
2351   // allocator which is updated via ReconfigureEncoder (however, note that the
2352   // actual encoder should only be reconfigured if needed).
2353   bool reconfigure_encoder = new_param ||
2354                              (new_parameters.encodings[0].bitrate_priority !=
2355                               rtp_parameters_.encodings[0].bitrate_priority) ||
2356                              new_parameters.encodings[0].scalability_mode !=
2357                                  rtp_parameters_.encodings[0].scalability_mode;
2358 
2359   // Note that the simulcast encoder adapter relies on the fact that layers
2360   // de/activation triggers encoder reinitialization.
2361   bool new_send_state = false;
2362   for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
2363     bool new_active = IsLayerActive(new_parameters.encodings[i]);
2364     bool old_active = IsLayerActive(rtp_parameters_.encodings[i]);
2365     if (new_active != old_active) {
2366       new_send_state = true;
2367     }
2368   }
2369   rtp_parameters_ = new_parameters;
2370   // Codecs are currently handled at the WebRtcVideoChannel level.
2371   rtp_parameters_.codecs.clear();
2372   if (reconfigure_encoder || new_send_state) {
2373     // Callback responsibility is delegated to ReconfigureEncoder()
2374     ReconfigureEncoder(std::move(callback));
2375     callback = nullptr;
2376   }
2377   if (new_send_state) {
2378     UpdateSendState();
2379   }
2380   if (new_degradation_preference) {
2381     if (source_ && stream_) {
2382       stream_->SetSource(source_, GetDegradationPreference());
2383     }
2384   }
2385   return webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
2386 }
2387 
2388 webrtc::RtpParameters
GetRtpParameters() const2389 WebRtcVideoChannel::WebRtcVideoSendStream::GetRtpParameters() const {
2390   RTC_DCHECK_RUN_ON(&thread_checker_);
2391   return rtp_parameters_;
2392 }
2393 
SetFrameEncryptor(rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor)2394 void WebRtcVideoChannel::WebRtcVideoSendStream::SetFrameEncryptor(
2395     rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
2396   RTC_DCHECK_RUN_ON(&thread_checker_);
2397   parameters_.config.frame_encryptor = frame_encryptor;
2398   if (stream_) {
2399     RTC_LOG(LS_INFO)
2400         << "RecreateWebRtcStream (send) because of SetFrameEncryptor, ssrc="
2401         << parameters_.config.rtp.ssrcs[0];
2402     RecreateWebRtcStream();
2403   }
2404 }
2405 
SetEncoderSelector(webrtc::VideoEncoderFactory::EncoderSelectorInterface * encoder_selector)2406 void WebRtcVideoChannel::WebRtcVideoSendStream::SetEncoderSelector(
2407     webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
2408   RTC_DCHECK_RUN_ON(&thread_checker_);
2409   parameters_.config.encoder_selector = encoder_selector;
2410   if (stream_) {
2411     RTC_LOG(LS_INFO)
2412         << "RecreateWebRtcStream (send) because of SetEncoderSelector, ssrc="
2413         << parameters_.config.rtp.ssrcs[0];
2414     RecreateWebRtcStream();
2415   }
2416 }
2417 
UpdateSendState()2418 void WebRtcVideoChannel::WebRtcVideoSendStream::UpdateSendState() {
2419   RTC_DCHECK_RUN_ON(&thread_checker_);
2420   if (sending_) {
2421     RTC_DCHECK(stream_ != nullptr);
2422     size_t num_layers = rtp_parameters_.encodings.size();
2423     if (parameters_.encoder_config.number_of_streams == 1) {
2424       // SVC is used. Only one simulcast layer is present.
2425       num_layers = 1;
2426     }
2427     std::vector<bool> active_layers(num_layers);
2428     for (size_t i = 0; i < num_layers; ++i) {
2429       active_layers[i] = IsLayerActive(rtp_parameters_.encodings[i]);
2430     }
2431     if (parameters_.encoder_config.number_of_streams == 1 &&
2432         rtp_parameters_.encodings.size() > 1) {
2433       // SVC is used.
2434       // The only present simulcast layer should be active if any of the
2435       // configured SVC layers is active.
2436       active_layers[0] =
2437           absl::c_any_of(rtp_parameters_.encodings,
2438                          [](const auto& encoding) { return encoding.active; });
2439     }
2440     // This updates what simulcast layers are sending, and possibly starts
2441     // or stops the VideoSendStream.
2442     stream_->StartPerRtpStream(active_layers);
2443   } else {
2444     if (stream_ != nullptr) {
2445       stream_->Stop();
2446     }
2447   }
2448 }
2449 
2450 webrtc::VideoEncoderConfig
CreateVideoEncoderConfig(const VideoCodec & codec) const2451 WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(
2452     const VideoCodec& codec) const {
2453   RTC_DCHECK_RUN_ON(&thread_checker_);
2454   webrtc::VideoEncoderConfig encoder_config;
2455   encoder_config.codec_type = webrtc::PayloadStringToCodecType(codec.name);
2456   encoder_config.video_format =
2457       webrtc::SdpVideoFormat(codec.name, codec.params);
2458 
2459   bool is_screencast = parameters_.options.is_screencast.value_or(false);
2460   if (is_screencast) {
2461     encoder_config.min_transmit_bitrate_bps =
2462         1000 * parameters_.options.screencast_min_bitrate_kbps.value_or(0);
2463     encoder_config.content_type =
2464         webrtc::VideoEncoderConfig::ContentType::kScreen;
2465   } else {
2466     encoder_config.min_transmit_bitrate_bps = 0;
2467     encoder_config.content_type =
2468         webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
2469   }
2470 
2471   // By default, the stream count for the codec configuration should match the
2472   // number of negotiated ssrcs. But if the codec is disabled for simulcast
2473   // or a screencast (and not in simulcast screenshare experiment), only
2474   // configure a single stream.
2475   encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
2476   if (IsCodecDisabledForSimulcast(codec.name, call_->trials())) {
2477     encoder_config.number_of_streams = 1;
2478   }
2479 
2480   // parameters_.max_bitrate comes from the max bitrate set at the SDP
2481   // (m-section) level with the attribute "b=AS." Note that we override this
2482   // value below if the RtpParameters max bitrate set with
2483   // RtpSender::SetParameters has a lower value.
2484   int stream_max_bitrate = parameters_.max_bitrate_bps;
2485   // When simulcast is enabled (when there are multiple encodings),
2486   // encodings[i].max_bitrate_bps will be enforced by
2487   // encoder_config.simulcast_layers[i].max_bitrate_bps. Otherwise, it's
2488   // enforced by stream_max_bitrate, taking the minimum of the two maximums
2489   // (one coming from SDP, the other coming from RtpParameters).
2490   if (rtp_parameters_.encodings[0].max_bitrate_bps &&
2491       rtp_parameters_.encodings.size() == 1) {
2492     stream_max_bitrate =
2493         MinPositive(*(rtp_parameters_.encodings[0].max_bitrate_bps),
2494                     parameters_.max_bitrate_bps);
2495   }
2496 
2497   // The codec max bitrate comes from the "x-google-max-bitrate" parameter
2498   // attribute set in the SDP for a specific codec. As done in
2499   // WebRtcVideoChannel::SetSendParameters, this value does not override the
2500   // stream max_bitrate set above.
2501   int codec_max_bitrate_kbps;
2502   if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps) &&
2503       stream_max_bitrate == -1) {
2504     stream_max_bitrate = codec_max_bitrate_kbps * 1000;
2505   }
2506   encoder_config.max_bitrate_bps = stream_max_bitrate;
2507 
2508   // The encoder config's default bitrate priority is set to 1.0,
2509   // unless it is set through the sender's encoding parameters.
2510   // The bitrate priority, which is used in the bitrate allocation, is done
2511   // on a per sender basis, so we use the first encoding's value.
2512   encoder_config.bitrate_priority =
2513       rtp_parameters_.encodings[0].bitrate_priority;
2514 
2515   // Application-controlled state is held in the encoder_config's
2516   // simulcast_layers. Currently this is used to control which simulcast layers
2517   // are active and for configuring the min/max bitrate and max framerate.
2518   // The encoder_config's simulcast_layers is also used for non-simulcast (when
2519   // there is a single layer).
2520   RTC_DCHECK_GE(rtp_parameters_.encodings.size(),
2521                 encoder_config.number_of_streams);
2522   RTC_DCHECK_GT(encoder_config.number_of_streams, 0);
2523 
2524   // Copy all provided constraints.
2525   encoder_config.simulcast_layers.resize(rtp_parameters_.encodings.size());
2526   for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
2527     encoder_config.simulcast_layers[i].active =
2528         rtp_parameters_.encodings[i].active;
2529     encoder_config.simulcast_layers[i].scalability_mode =
2530         webrtc::ScalabilityModeFromString(
2531             rtp_parameters_.encodings[i].scalability_mode.value_or(""));
2532     if (rtp_parameters_.encodings[i].min_bitrate_bps) {
2533       encoder_config.simulcast_layers[i].min_bitrate_bps =
2534           *rtp_parameters_.encodings[i].min_bitrate_bps;
2535     }
2536     if (rtp_parameters_.encodings[i].max_bitrate_bps) {
2537       encoder_config.simulcast_layers[i].max_bitrate_bps =
2538           *rtp_parameters_.encodings[i].max_bitrate_bps;
2539     }
2540     if (rtp_parameters_.encodings[i].max_framerate) {
2541       encoder_config.simulcast_layers[i].max_framerate =
2542           *rtp_parameters_.encodings[i].max_framerate;
2543     }
2544     if (rtp_parameters_.encodings[i].scale_resolution_down_by) {
2545       encoder_config.simulcast_layers[i].scale_resolution_down_by =
2546           *rtp_parameters_.encodings[i].scale_resolution_down_by;
2547     }
2548     if (rtp_parameters_.encodings[i].num_temporal_layers) {
2549       encoder_config.simulcast_layers[i].num_temporal_layers =
2550           *rtp_parameters_.encodings[i].num_temporal_layers;
2551     }
2552     encoder_config.simulcast_layers[i].requested_resolution =
2553         rtp_parameters_.encodings[i].requested_resolution;
2554   }
2555 
2556   encoder_config.legacy_conference_mode = parameters_.conference_mode;
2557 
2558   encoder_config.is_quality_scaling_allowed =
2559       !disable_automatic_resize_ && !is_screencast &&
2560       (parameters_.config.rtp.ssrcs.size() == 1 ||
2561        NumActiveStreams(rtp_parameters_) == 1);
2562 
2563   // Ensure frame dropping is always enabled.
2564   encoder_config.frame_drop_enabled = true;
2565 
2566   int max_qp = kDefaultQpMax;
2567   codec.GetParam(kCodecParamMaxQuantization, &max_qp);
2568   encoder_config.max_qp = max_qp;
2569 
2570   return encoder_config;
2571 }
2572 
ReconfigureEncoder(webrtc::SetParametersCallback callback)2573 void WebRtcVideoChannel::WebRtcVideoSendStream::ReconfigureEncoder(
2574     webrtc::SetParametersCallback callback) {
2575   RTC_DCHECK_RUN_ON(&thread_checker_);
2576   if (!stream_) {
2577     // The webrtc::VideoSendStream `stream_` has not yet been created but other
2578     // parameters has changed.
2579     webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
2580     return;
2581   }
2582 
2583   RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
2584 
2585   RTC_CHECK(parameters_.codec_settings);
2586   VideoCodecSettings codec_settings = *parameters_.codec_settings;
2587 
2588   webrtc::VideoEncoderConfig encoder_config =
2589       CreateVideoEncoderConfig(codec_settings.codec);
2590 
2591   encoder_config.encoder_specific_settings =
2592       ConfigureVideoEncoderSettings(codec_settings.codec);
2593 
2594   stream_->ReconfigureVideoEncoder(encoder_config.Copy(), std::move(callback));
2595 
2596   encoder_config.encoder_specific_settings = NULL;
2597 
2598   parameters_.encoder_config = std::move(encoder_config);
2599 }
2600 
SetSend(bool send)2601 void WebRtcVideoChannel::WebRtcVideoSendStream::SetSend(bool send) {
2602   RTC_DCHECK_RUN_ON(&thread_checker_);
2603   sending_ = send;
2604   UpdateSendState();
2605 }
2606 
2607 std::vector<VideoSenderInfo>
GetPerLayerVideoSenderInfos(bool log_stats)2608 WebRtcVideoChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos(
2609     bool log_stats) {
2610   RTC_DCHECK_RUN_ON(&thread_checker_);
2611   VideoSenderInfo common_info;
2612   if (parameters_.codec_settings) {
2613     common_info.codec_name = parameters_.codec_settings->codec.name;
2614     common_info.codec_payload_type = parameters_.codec_settings->codec.id;
2615   }
2616   std::vector<VideoSenderInfo> infos;
2617   webrtc::VideoSendStream::Stats stats;
2618   if (stream_ == nullptr) {
2619     for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2620       common_info.add_ssrc(ssrc);
2621     }
2622     infos.push_back(common_info);
2623     return infos;
2624   } else {
2625     stats = stream_->GetStats();
2626     if (log_stats)
2627       RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
2628 
2629     // Metrics that are in common for all substreams.
2630     common_info.adapt_changes = stats.number_of_cpu_adapt_changes;
2631     common_info.adapt_reason =
2632         stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE;
2633     common_info.has_entered_low_resolution = stats.has_entered_low_resolution;
2634 
2635     // Get bandwidth limitation info from stream_->GetStats().
2636     // Input resolution (output from video_adapter) can be further scaled down
2637     // or higher video layer(s) can be dropped due to bitrate constraints.
2638     // Note, adapt_changes only include changes from the video_adapter.
2639     if (stats.bw_limited_resolution)
2640       common_info.adapt_reason |= ADAPTREASON_BANDWIDTH;
2641 
2642     common_info.quality_limitation_reason = stats.quality_limitation_reason;
2643     common_info.quality_limitation_durations_ms =
2644         stats.quality_limitation_durations_ms;
2645     common_info.quality_limitation_resolution_changes =
2646         stats.quality_limitation_resolution_changes;
2647     common_info.encoder_implementation_name = stats.encoder_implementation_name;
2648     common_info.target_bitrate = stats.target_media_bitrate_bps;
2649     common_info.ssrc_groups = ssrc_groups_;
2650     common_info.frames = stats.frames;
2651     common_info.framerate_input = stats.input_frame_rate;
2652     common_info.avg_encode_ms = stats.avg_encode_time_ms;
2653     common_info.encode_usage_percent = stats.encode_usage_percent;
2654     common_info.nominal_bitrate = stats.media_bitrate_bps;
2655     common_info.content_type = stats.content_type;
2656     common_info.aggregated_framerate_sent = stats.encode_frame_rate;
2657     common_info.aggregated_huge_frames_sent = stats.huge_frames_sent;
2658     common_info.power_efficient_encoder = stats.power_efficient_encoder;
2659 
2660     // The normal case is that substreams are present, handled below. But if
2661     // substreams are missing (can happen before negotiated/connected where we
2662     // have no stats yet) a single outbound-rtp is created representing any and
2663     // all layers.
2664     if (stats.substreams.empty()) {
2665       for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2666         common_info.add_ssrc(ssrc);
2667       }
2668       common_info.active =
2669           IsActiveFromEncodings(absl::nullopt, rtp_parameters_.encodings);
2670       common_info.framerate_sent = stats.encode_frame_rate;
2671       common_info.frames_encoded = stats.frames_encoded;
2672       common_info.total_encode_time_ms = stats.total_encode_time_ms;
2673       common_info.total_encoded_bytes_target = stats.total_encoded_bytes_target;
2674       common_info.frames_sent = stats.frames_encoded;
2675       common_info.huge_frames_sent = stats.huge_frames_sent;
2676       infos.push_back(common_info);
2677       return infos;
2678     }
2679   }
2680   // Merge `stats.substreams`, which may contain additional SSRCs for RTX or
2681   // Flexfec, with media SSRCs. This results in a set of substreams that match
2682   // with the outbound-rtp stats objects.
2683   auto outbound_rtp_substreams =
2684       MergeInfoAboutOutboundRtpSubstreams(stats.substreams);
2685   // If SVC is used, one stream is configured but multiple encodings exist. This
2686   // is not spec-compliant, but it is how we've implemented SVC so this affects
2687   // how the RTP stream's "active" value is determined.
2688   bool is_svc = (parameters_.encoder_config.number_of_streams == 1 &&
2689                  rtp_parameters_.encodings.size() > 1);
2690   for (const auto& pair : outbound_rtp_substreams) {
2691     auto info = common_info;
2692     uint32_t ssrc = pair.first;
2693     info.add_ssrc(ssrc);
2694     info.rid = parameters_.config.rtp.GetRidForSsrc(ssrc);
2695     info.active = IsActiveFromEncodings(
2696         !is_svc ? absl::optional<uint32_t>(ssrc) : absl::nullopt,
2697         rtp_parameters_.encodings);
2698     auto stream_stats = pair.second;
2699     RTC_DCHECK_EQ(stream_stats.type,
2700                   webrtc::VideoSendStream::StreamStats::StreamType::kMedia);
2701     info.payload_bytes_sent = stream_stats.rtp_stats.transmitted.payload_bytes;
2702     info.header_and_padding_bytes_sent =
2703         stream_stats.rtp_stats.transmitted.header_bytes +
2704         stream_stats.rtp_stats.transmitted.padding_bytes;
2705     info.packets_sent = stream_stats.rtp_stats.transmitted.packets;
2706     info.total_packet_send_delay +=
2707         stream_stats.rtp_stats.transmitted.total_packet_delay;
2708     info.send_frame_width = stream_stats.width;
2709     info.send_frame_height = stream_stats.height;
2710     info.key_frames_encoded = stream_stats.frame_counts.key_frames;
2711     info.framerate_sent = stream_stats.encode_frame_rate;
2712     info.frames_encoded = stream_stats.frames_encoded;
2713     info.frames_sent = stream_stats.frames_encoded;
2714     info.retransmitted_bytes_sent =
2715         stream_stats.rtp_stats.retransmitted.payload_bytes;
2716     info.retransmitted_packets_sent =
2717         stream_stats.rtp_stats.retransmitted.packets;
2718     info.firs_rcvd = stream_stats.rtcp_packet_type_counts.fir_packets;
2719     info.nacks_rcvd = stream_stats.rtcp_packet_type_counts.nack_packets;
2720     info.plis_rcvd = stream_stats.rtcp_packet_type_counts.pli_packets;
2721     if (stream_stats.report_block_data.has_value()) {
2722       info.packets_lost =
2723           stream_stats.report_block_data->report_block().packets_lost;
2724       info.fraction_lost =
2725           static_cast<float>(
2726               stream_stats.report_block_data->report_block().fraction_lost) /
2727           (1 << 8);
2728       info.report_block_datas.push_back(*stream_stats.report_block_data);
2729     }
2730     info.qp_sum = stream_stats.qp_sum;
2731     info.total_encode_time_ms = stream_stats.total_encode_time_ms;
2732     info.total_encoded_bytes_target = stream_stats.total_encoded_bytes_target;
2733     info.huge_frames_sent = stream_stats.huge_frames_sent;
2734     infos.push_back(info);
2735   }
2736   return infos;
2737 }
2738 
2739 VideoSenderInfo
GetAggregatedVideoSenderInfo(const std::vector<VideoSenderInfo> & infos) const2740 WebRtcVideoChannel::WebRtcVideoSendStream::GetAggregatedVideoSenderInfo(
2741     const std::vector<VideoSenderInfo>& infos) const {
2742   RTC_DCHECK_RUN_ON(&thread_checker_);
2743   RTC_CHECK(!infos.empty());
2744   if (infos.size() == 1) {
2745     return infos[0];
2746   }
2747   VideoSenderInfo info = infos[0];
2748   info.local_stats.clear();
2749   for (uint32_t ssrc : parameters_.config.rtp.ssrcs) {
2750     info.add_ssrc(ssrc);
2751   }
2752   info.framerate_sent = info.aggregated_framerate_sent;
2753   info.huge_frames_sent = info.aggregated_huge_frames_sent;
2754 
2755   for (size_t i = 1; i < infos.size(); i++) {
2756     info.key_frames_encoded += infos[i].key_frames_encoded;
2757     info.payload_bytes_sent += infos[i].payload_bytes_sent;
2758     info.header_and_padding_bytes_sent +=
2759         infos[i].header_and_padding_bytes_sent;
2760     info.packets_sent += infos[i].packets_sent;
2761     info.total_packet_send_delay += infos[i].total_packet_send_delay;
2762     info.retransmitted_bytes_sent += infos[i].retransmitted_bytes_sent;
2763     info.retransmitted_packets_sent += infos[i].retransmitted_packets_sent;
2764     info.packets_lost += infos[i].packets_lost;
2765     if (infos[i].send_frame_width > info.send_frame_width)
2766       info.send_frame_width = infos[i].send_frame_width;
2767     if (infos[i].send_frame_height > info.send_frame_height)
2768       info.send_frame_height = infos[i].send_frame_height;
2769     info.firs_rcvd += infos[i].firs_rcvd;
2770     info.nacks_rcvd += infos[i].nacks_rcvd;
2771     info.plis_rcvd += infos[i].plis_rcvd;
2772     if (infos[i].report_block_datas.size())
2773       info.report_block_datas.push_back(infos[i].report_block_datas[0]);
2774     if (infos[i].qp_sum) {
2775       if (!info.qp_sum) {
2776         info.qp_sum = 0;
2777       }
2778       info.qp_sum = *info.qp_sum + *infos[i].qp_sum;
2779     }
2780     info.frames_encoded += infos[i].frames_encoded;
2781     info.frames_sent += infos[i].frames_sent;
2782     info.total_encode_time_ms += infos[i].total_encode_time_ms;
2783     info.total_encoded_bytes_target += infos[i].total_encoded_bytes_target;
2784   }
2785   return info;
2786 }
2787 
FillBitrateInfo(BandwidthEstimationInfo * bwe_info)2788 void WebRtcVideoChannel::WebRtcVideoSendStream::FillBitrateInfo(
2789     BandwidthEstimationInfo* bwe_info) {
2790   RTC_DCHECK_RUN_ON(&thread_checker_);
2791   if (stream_ == NULL) {
2792     return;
2793   }
2794   webrtc::VideoSendStream::Stats stats = stream_->GetStats();
2795   for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it =
2796            stats.substreams.begin();
2797        it != stats.substreams.end(); ++it) {
2798     bwe_info->transmit_bitrate += it->second.total_bitrate_bps;
2799     bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps;
2800   }
2801   bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps;
2802   bwe_info->actual_enc_bitrate += stats.media_bitrate_bps;
2803 }
2804 
2805 void WebRtcVideoChannel::WebRtcVideoSendStream::
SetEncoderToPacketizerFrameTransformer(rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)2806     SetEncoderToPacketizerFrameTransformer(
2807         rtc::scoped_refptr<webrtc::FrameTransformerInterface>
2808             frame_transformer) {
2809   RTC_DCHECK_RUN_ON(&thread_checker_);
2810   parameters_.config.frame_transformer = std::move(frame_transformer);
2811   if (stream_)
2812     RecreateWebRtcStream();
2813 }
2814 
RecreateWebRtcStream()2815 void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
2816   RTC_DCHECK_RUN_ON(&thread_checker_);
2817   if (stream_ != NULL) {
2818     call_->DestroyVideoSendStream(stream_);
2819   }
2820 
2821   RTC_CHECK(parameters_.codec_settings);
2822   RTC_DCHECK_EQ((parameters_.encoder_config.content_type ==
2823                  webrtc::VideoEncoderConfig::ContentType::kScreen),
2824                 parameters_.options.is_screencast.value_or(false))
2825       << "encoder content type inconsistent with screencast option";
2826   parameters_.encoder_config.encoder_specific_settings =
2827       ConfigureVideoEncoderSettings(parameters_.codec_settings->codec);
2828 
2829   webrtc::VideoSendStream::Config config = parameters_.config.Copy();
2830   if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) {
2831     RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
2832                            "payload type the set codec. Ignoring RTX.";
2833     config.rtp.rtx.ssrcs.clear();
2834   }
2835   if (parameters_.encoder_config.number_of_streams == 1) {
2836     // SVC is used instead of simulcast. Remove unnecessary SSRCs.
2837     if (config.rtp.ssrcs.size() > 1) {
2838       config.rtp.ssrcs.resize(1);
2839       if (config.rtp.rtx.ssrcs.size() > 1) {
2840         config.rtp.rtx.ssrcs.resize(1);
2841       }
2842     }
2843   }
2844   stream_ = call_->CreateVideoSendStream(std::move(config),
2845                                          parameters_.encoder_config.Copy());
2846 
2847   parameters_.encoder_config.encoder_specific_settings = NULL;
2848 
2849   // Calls stream_->StartPerRtpStream() to start the VideoSendStream
2850   // if necessary conditions are met.
2851   UpdateSendState();
2852 
2853   // Attach the source after starting the send stream to prevent frames from
2854   // being injected into a not-yet initializated video stream encoder.
2855   if (source_) {
2856     stream_->SetSource(source_, GetDegradationPreference());
2857   }
2858 }
2859 
GenerateKeyFrame(const std::vector<std::string> & rids)2860 void WebRtcVideoChannel::WebRtcVideoSendStream::GenerateKeyFrame(
2861     const std::vector<std::string>& rids) {
2862   RTC_DCHECK_RUN_ON(&thread_checker_);
2863   if (stream_ != NULL) {
2864     stream_->GenerateKeyFrame(rids);
2865   } else {
2866     RTC_LOG(LS_WARNING)
2867         << "Absent send stream; ignoring request to generate keyframe.";
2868   }
2869 }
2870 
WebRtcVideoReceiveStream(WebRtcVideoChannel * channel,webrtc::Call * call,const StreamParams & sp,webrtc::VideoReceiveStreamInterface::Config config,bool default_stream,const std::vector<VideoCodecSettings> & recv_codecs,const webrtc::FlexfecReceiveStream::Config & flexfec_config)2871 WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
2872     WebRtcVideoChannel* channel,
2873     webrtc::Call* call,
2874     const StreamParams& sp,
2875     webrtc::VideoReceiveStreamInterface::Config config,
2876     bool default_stream,
2877     const std::vector<VideoCodecSettings>& recv_codecs,
2878     const webrtc::FlexfecReceiveStream::Config& flexfec_config)
2879     : channel_(channel),
2880       call_(call),
2881       stream_params_(sp),
2882       stream_(NULL),
2883       default_stream_(default_stream),
2884       config_(std::move(config)),
2885       flexfec_config_(flexfec_config),
2886       flexfec_stream_(nullptr),
2887       sink_(NULL),
2888       first_frame_timestamp_(-1),
2889       estimated_remote_start_ntp_time_ms_(0) {
2890   RTC_DCHECK(config_.decoder_factory);
2891   RTC_DCHECK(config_.decoders.empty())
2892       << "Decoder info is supplied via `recv_codecs`";
2893 
2894   ExtractCodecInformation(recv_codecs, config_.rtp.rtx_associated_payload_types,
2895                           config_.rtp.raw_payload_types, config_.decoders);
2896   const VideoCodecSettings& codec = recv_codecs.front();
2897   config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type;
2898   config_.rtp.red_payload_type = codec.ulpfec.red_payload_type;
2899   config_.rtp.lntf.enabled = HasLntf(codec.codec);
2900   config_.rtp.nack.rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0;
2901   if (codec.rtx_time != -1 && config_.rtp.nack.rtp_history_ms != 0) {
2902     config_.rtp.nack.rtp_history_ms = codec.rtx_time;
2903   }
2904 
2905   config_.rtp.rtcp_xr.receiver_reference_time_report = HasRrtr(codec.codec);
2906 
2907   if (codec.ulpfec.red_rtx_payload_type != -1) {
2908     config_.rtp
2909         .rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] =
2910         codec.ulpfec.red_payload_type;
2911   }
2912 
2913   config_.renderer = this;
2914   flexfec_config_.payload_type = flexfec_config.payload_type;
2915 
2916   CreateReceiveStream();
2917   StartReceiveStream();
2918 }
2919 
~WebRtcVideoReceiveStream()2920 WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
2921   call_->DestroyVideoReceiveStream(stream_);
2922   if (flexfec_stream_)
2923     call_->DestroyFlexfecReceiveStream(flexfec_stream_);
2924 }
2925 
2926 webrtc::VideoReceiveStreamInterface&
stream()2927 WebRtcVideoChannel::WebRtcVideoReceiveStream::stream() {
2928   RTC_DCHECK(stream_);
2929   return *stream_;
2930 }
2931 
2932 webrtc::FlexfecReceiveStream*
flexfec_stream()2933 WebRtcVideoChannel::WebRtcVideoReceiveStream::flexfec_stream() {
2934   return flexfec_stream_;
2935 }
2936 
2937 const std::vector<uint32_t>&
GetSsrcs() const2938 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSsrcs() const {
2939   return stream_params_.ssrcs;
2940 }
2941 
2942 std::vector<webrtc::RtpSource>
GetSources()2943 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSources() {
2944   RTC_DCHECK(stream_);
2945   return stream_->GetSources();
2946 }
2947 
2948 webrtc::RtpParameters
GetRtpParameters() const2949 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetRtpParameters() const {
2950   webrtc::RtpParameters rtp_parameters;
2951 
2952   std::vector<uint32_t> primary_ssrcs;
2953   stream_params_.GetPrimarySsrcs(&primary_ssrcs);
2954   for (uint32_t ssrc : primary_ssrcs) {
2955     rtp_parameters.encodings.emplace_back();
2956     rtp_parameters.encodings.back().ssrc = ssrc;
2957   }
2958 
2959   rtp_parameters.header_extensions = config_.rtp.extensions;
2960   rtp_parameters.rtcp.reduced_size =
2961       config_.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
2962 
2963   return rtp_parameters;
2964 }
2965 
ReconfigureCodecs(const std::vector<VideoCodecSettings> & recv_codecs)2966 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::ReconfigureCodecs(
2967     const std::vector<VideoCodecSettings>& recv_codecs) {
2968   RTC_DCHECK(stream_);
2969   RTC_DCHECK(!recv_codecs.empty());
2970 
2971   std::map<int, int> rtx_associated_payload_types;
2972   std::set<int> raw_payload_types;
2973   std::vector<webrtc::VideoReceiveStreamInterface::Decoder> decoders;
2974   ExtractCodecInformation(recv_codecs, rtx_associated_payload_types,
2975                           raw_payload_types, decoders);
2976 
2977   const auto& codec = recv_codecs.front();
2978 
2979   if (config_.rtp.red_payload_type != codec.ulpfec.red_payload_type ||
2980       config_.rtp.ulpfec_payload_type != codec.ulpfec.ulpfec_payload_type) {
2981     config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type;
2982     config_.rtp.red_payload_type = codec.ulpfec.red_payload_type;
2983     stream_->SetProtectionPayloadTypes(config_.rtp.red_payload_type,
2984                                        config_.rtp.ulpfec_payload_type);
2985   }
2986 
2987   const bool has_lntf = HasLntf(codec.codec);
2988   if (config_.rtp.lntf.enabled != has_lntf) {
2989     config_.rtp.lntf.enabled = has_lntf;
2990     stream_->SetLossNotificationEnabled(has_lntf);
2991   }
2992 
2993   int new_history_ms = config_.rtp.nack.rtp_history_ms;
2994   const int rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0;
2995   if (rtp_history_ms != config_.rtp.nack.rtp_history_ms) {
2996     new_history_ms = rtp_history_ms;
2997   }
2998 
2999   // The rtx-time parameter can be used to override the hardcoded default for
3000   // the NACK buffer length.
3001   if (codec.rtx_time != -1 && new_history_ms != 0) {
3002     new_history_ms = codec.rtx_time;
3003   }
3004 
3005   if (config_.rtp.nack.rtp_history_ms != new_history_ms) {
3006     config_.rtp.nack.rtp_history_ms = new_history_ms;
3007     stream_->SetNackHistory(webrtc::TimeDelta::Millis(new_history_ms));
3008   }
3009 
3010   const bool has_rtr = HasRrtr(codec.codec);
3011   if (has_rtr != config_.rtp.rtcp_xr.receiver_reference_time_report) {
3012     config_.rtp.rtcp_xr.receiver_reference_time_report = has_rtr;
3013     stream_->SetRtcpXr(config_.rtp.rtcp_xr);
3014   }
3015 
3016   if (codec.ulpfec.red_rtx_payload_type != -1) {
3017     rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] =
3018         codec.ulpfec.red_payload_type;
3019   }
3020 
3021   if (config_.rtp.rtx_associated_payload_types !=
3022       rtx_associated_payload_types) {
3023     stream_->SetAssociatedPayloadTypes(rtx_associated_payload_types);
3024     rtx_associated_payload_types.swap(config_.rtp.rtx_associated_payload_types);
3025   }
3026 
3027   bool recreate_needed = false;
3028 
3029   if (raw_payload_types != config_.rtp.raw_payload_types) {
3030     raw_payload_types.swap(config_.rtp.raw_payload_types);
3031     recreate_needed = true;
3032   }
3033 
3034   if (decoders != config_.decoders) {
3035     decoders.swap(config_.decoders);
3036     recreate_needed = true;
3037   }
3038 
3039   return recreate_needed;
3040 }
3041 
SetFeedbackParameters(bool lntf_enabled,bool nack_enabled,bool transport_cc_enabled,webrtc::RtcpMode rtcp_mode,int rtx_time)3042 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFeedbackParameters(
3043     bool lntf_enabled,
3044     bool nack_enabled,
3045     bool transport_cc_enabled,
3046     webrtc::RtcpMode rtcp_mode,
3047     int rtx_time) {
3048   RTC_DCHECK(stream_);
3049 
3050   if (config_.rtp.rtcp_mode != rtcp_mode) {
3051     config_.rtp.rtcp_mode = rtcp_mode;
3052     stream_->SetRtcpMode(rtcp_mode);
3053 
3054     flexfec_config_.rtcp_mode = rtcp_mode;
3055     if (flexfec_stream_) {
3056       flexfec_stream_->SetRtcpMode(rtcp_mode);
3057     }
3058   }
3059 
3060   if (config_.rtp.transport_cc != transport_cc_enabled) {
3061     config_.rtp.transport_cc = transport_cc_enabled;
3062     stream_->SetTransportCc(transport_cc_enabled);
3063     // TODO(brandtr): We should be spec-compliant and set `transport_cc` here
3064     // based on the rtcp-fb for the FlexFEC codec, not the media codec.
3065     flexfec_config_.rtp.transport_cc = transport_cc_enabled;
3066     if (flexfec_stream_) {
3067       flexfec_stream_->SetTransportCc(transport_cc_enabled);
3068     }
3069   }
3070 
3071   config_.rtp.lntf.enabled = lntf_enabled;
3072   stream_->SetLossNotificationEnabled(lntf_enabled);
3073 
3074   int nack_history_ms =
3075       nack_enabled ? rtx_time != -1 ? rtx_time : kNackHistoryMs : 0;
3076   config_.rtp.nack.rtp_history_ms = nack_history_ms;
3077   stream_->SetNackHistory(webrtc::TimeDelta::Millis(nack_history_ms));
3078 }
3079 
SetFlexFecPayload(int payload_type)3080 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFlexFecPayload(
3081     int payload_type) {
3082   // TODO(bugs.webrtc.org/11993, tommi): See if it is better to always have a
3083   // flexfec stream object around and instead of recreating the video stream,
3084   // reconfigure the flexfec object from within the rtp callback (soon to be on
3085   // the network thread).
3086   if (flexfec_stream_) {
3087     if (flexfec_stream_->payload_type() == payload_type) {
3088       RTC_DCHECK_EQ(flexfec_config_.payload_type, payload_type);
3089       return;
3090     }
3091 
3092     flexfec_config_.payload_type = payload_type;
3093     flexfec_stream_->SetPayloadType(payload_type);
3094 
3095     if (payload_type == -1) {
3096       stream_->SetFlexFecProtection(nullptr);
3097       call_->DestroyFlexfecReceiveStream(flexfec_stream_);
3098       flexfec_stream_ = nullptr;
3099     }
3100   } else if (payload_type != -1) {
3101     flexfec_config_.payload_type = payload_type;
3102     if (flexfec_config_.IsCompleteAndEnabled()) {
3103       flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
3104       stream_->SetFlexFecProtection(flexfec_stream_);
3105     }
3106   } else {
3107     // Noop. No flexfec stream exists and "new" payload_type == -1.
3108     RTC_DCHECK(!flexfec_config_.IsCompleteAndEnabled());
3109     flexfec_config_.payload_type = payload_type;
3110   }
3111 }
3112 
SetRecvParameters(const ChangedRecvParameters & params)3113 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
3114     const ChangedRecvParameters& params) {
3115   RTC_DCHECK(stream_);
3116   bool video_needs_recreation = false;
3117   if (params.codec_settings) {
3118     video_needs_recreation = ReconfigureCodecs(*params.codec_settings);
3119   }
3120 
3121   if (params.rtp_header_extensions) {
3122     if (config_.rtp.extensions != *params.rtp_header_extensions) {
3123       config_.rtp.extensions = *params.rtp_header_extensions;
3124       stream_->SetRtpExtensions(config_.rtp.extensions);
3125     }
3126 
3127     if (flexfec_config_.rtp.extensions != *params.rtp_header_extensions) {
3128       flexfec_config_.rtp.extensions = *params.rtp_header_extensions;
3129       if (flexfec_stream_) {
3130         flexfec_stream_->SetRtpExtensions(flexfec_config_.rtp.extensions);
3131       }
3132     }
3133   }
3134 
3135   if (params.flexfec_payload_type)
3136     SetFlexFecPayload(*params.flexfec_payload_type);
3137 
3138   if (video_needs_recreation) {
3139     RecreateReceiveStream();
3140   } else {
3141     RTC_DLOG_F(LS_INFO) << "No receive stream recreate needed.";
3142   }
3143 }
3144 
RecreateReceiveStream()3145 void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() {
3146   RTC_DCHECK(stream_);
3147   absl::optional<int> base_minimum_playout_delay_ms;
3148   absl::optional<webrtc::VideoReceiveStreamInterface::RecordingState>
3149       recording_state;
3150   if (stream_) {
3151     base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs();
3152     recording_state = stream_->SetAndGetRecordingState(
3153         webrtc::VideoReceiveStreamInterface::RecordingState(),
3154         /*generate_key_frame=*/false);
3155     call_->DestroyVideoReceiveStream(stream_);
3156     stream_ = nullptr;
3157   }
3158 
3159   if (flexfec_stream_) {
3160     call_->DestroyFlexfecReceiveStream(flexfec_stream_);
3161     flexfec_stream_ = nullptr;
3162   }
3163 
3164   CreateReceiveStream();
3165 
3166   if (base_minimum_playout_delay_ms) {
3167     stream_->SetBaseMinimumPlayoutDelayMs(
3168         base_minimum_playout_delay_ms.value());
3169   }
3170   if (recording_state) {
3171     stream_->SetAndGetRecordingState(std::move(*recording_state),
3172                                      /*generate_key_frame=*/false);
3173   }
3174 
3175   StartReceiveStream();
3176 }
3177 
CreateReceiveStream()3178 void WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateReceiveStream() {
3179   RTC_DCHECK(!stream_);
3180   RTC_DCHECK(!flexfec_stream_);
3181   if (flexfec_config_.IsCompleteAndEnabled()) {
3182     flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
3183   }
3184 
3185   webrtc::VideoReceiveStreamInterface::Config config = config_.Copy();
3186   config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr);
3187   config.rtp.packet_sink_ = flexfec_stream_;
3188   stream_ = call_->CreateVideoReceiveStream(std::move(config));
3189 }
3190 
StartReceiveStream()3191 void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartReceiveStream() {
3192   stream_->Start();
3193   if (IsEnabled(call_->trials(), "WebRTC-Video-BufferPacketsWithUnknownSsrc")) {
3194     channel_->BackfillBufferedPackets(stream_params_.ssrcs);
3195   }
3196 }
3197 
OnFrame(const webrtc::VideoFrame & frame)3198 void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame(
3199     const webrtc::VideoFrame& frame) {
3200   webrtc::MutexLock lock(&sink_lock_);
3201 
3202   int64_t time_now_ms = rtc::TimeMillis();
3203   if (first_frame_timestamp_ < 0)
3204     first_frame_timestamp_ = time_now_ms;
3205   int64_t elapsed_time_ms = time_now_ms - first_frame_timestamp_;
3206   if (frame.ntp_time_ms() > 0)
3207     estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms;
3208 
3209   if (sink_ == NULL) {
3210     RTC_LOG(LS_WARNING)
3211         << "VideoReceiveStreamInterface not connected to a VideoSink.";
3212     return;
3213   }
3214 
3215   sink_->OnFrame(frame);
3216 }
3217 
IsDefaultStream() const3218 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::IsDefaultStream() const {
3219   return default_stream_;
3220 }
3221 
SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)3222 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFrameDecryptor(
3223     rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
3224   config_.frame_decryptor = frame_decryptor;
3225   if (stream_) {
3226     RTC_LOG(LS_INFO)
3227         << "Setting FrameDecryptor (recv) because of SetFrameDecryptor, "
3228            "remote_ssrc="
3229         << config_.rtp.remote_ssrc;
3230     stream_->SetFrameDecryptor(frame_decryptor);
3231   }
3232 }
3233 
SetBaseMinimumPlayoutDelayMs(int delay_ms)3234 bool WebRtcVideoChannel::WebRtcVideoReceiveStream::SetBaseMinimumPlayoutDelayMs(
3235     int delay_ms) {
3236   return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false;
3237 }
3238 
GetBaseMinimumPlayoutDelayMs() const3239 int WebRtcVideoChannel::WebRtcVideoReceiveStream::GetBaseMinimumPlayoutDelayMs()
3240     const {
3241   return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0;
3242 }
3243 
SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink)3244 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetSink(
3245     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
3246   webrtc::MutexLock lock(&sink_lock_);
3247   sink_ = sink;
3248 }
3249 
3250 VideoReceiverInfo
GetVideoReceiverInfo(bool log_stats)3251 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo(
3252     bool log_stats) {
3253   VideoReceiverInfo info;
3254   info.ssrc_groups = stream_params_.ssrc_groups;
3255   info.add_ssrc(config_.rtp.remote_ssrc);
3256   webrtc::VideoReceiveStreamInterface::Stats stats = stream_->GetStats();
3257   info.decoder_implementation_name = stats.decoder_implementation_name;
3258   info.power_efficient_decoder = stats.power_efficient_decoder;
3259   if (stats.current_payload_type != -1) {
3260     info.codec_payload_type = stats.current_payload_type;
3261     auto decoder_it = absl::c_find_if(config_.decoders, [&](const auto& d) {
3262       return d.payload_type == stats.current_payload_type;
3263     });
3264     if (decoder_it != config_.decoders.end())
3265       info.codec_name = decoder_it->video_format.name;
3266   }
3267   info.payload_bytes_rcvd = stats.rtp_stats.packet_counter.payload_bytes;
3268   info.header_and_padding_bytes_rcvd =
3269       stats.rtp_stats.packet_counter.header_bytes +
3270       stats.rtp_stats.packet_counter.padding_bytes;
3271   info.packets_rcvd = stats.rtp_stats.packet_counter.packets;
3272   info.packets_lost = stats.rtp_stats.packets_lost;
3273   info.jitter_ms = stats.rtp_stats.jitter / (kVideoCodecClockrate / 1000);
3274 
3275   info.framerate_rcvd = stats.network_frame_rate;
3276   info.framerate_decoded = stats.decode_frame_rate;
3277   info.framerate_output = stats.render_frame_rate;
3278   info.frame_width = stats.width;
3279   info.frame_height = stats.height;
3280 
3281   {
3282     webrtc::MutexLock frame_cs(&sink_lock_);
3283     info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_;
3284   }
3285 
3286   info.decode_ms = stats.decode_ms;
3287   info.max_decode_ms = stats.max_decode_ms;
3288   info.current_delay_ms = stats.current_delay_ms;
3289   info.target_delay_ms = stats.target_delay_ms;
3290   info.jitter_buffer_ms = stats.jitter_buffer_ms;
3291   info.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds;
3292   info.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count;
3293   info.min_playout_delay_ms = stats.min_playout_delay_ms;
3294   info.render_delay_ms = stats.render_delay_ms;
3295   info.frames_received =
3296       stats.frame_counts.key_frames + stats.frame_counts.delta_frames;
3297   info.frames_dropped = stats.frames_dropped;
3298   info.frames_decoded = stats.frames_decoded;
3299   info.key_frames_decoded = stats.frame_counts.key_frames;
3300   info.frames_rendered = stats.frames_rendered;
3301   info.qp_sum = stats.qp_sum;
3302   info.total_decode_time = stats.total_decode_time;
3303   info.total_processing_delay = stats.total_processing_delay;
3304   info.total_assembly_time = stats.total_assembly_time;
3305   info.frames_assembled_from_multiple_packets =
3306       stats.frames_assembled_from_multiple_packets;
3307   info.last_packet_received_timestamp_ms =
3308       stats.rtp_stats.last_packet_received_timestamp_ms;
3309   info.estimated_playout_ntp_timestamp_ms =
3310       stats.estimated_playout_ntp_timestamp_ms;
3311   info.first_frame_received_to_decoded_ms =
3312       stats.first_frame_received_to_decoded_ms;
3313   info.total_inter_frame_delay = stats.total_inter_frame_delay;
3314   info.total_squared_inter_frame_delay = stats.total_squared_inter_frame_delay;
3315   info.interframe_delay_max_ms = stats.interframe_delay_max_ms;
3316   info.freeze_count = stats.freeze_count;
3317   info.pause_count = stats.pause_count;
3318   info.total_freezes_duration_ms = stats.total_freezes_duration_ms;
3319   info.total_pauses_duration_ms = stats.total_pauses_duration_ms;
3320 
3321   info.content_type = stats.content_type;
3322 
3323   info.firs_sent = stats.rtcp_packet_type_counts.fir_packets;
3324   info.plis_sent = stats.rtcp_packet_type_counts.pli_packets;
3325   info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets;
3326   // TODO(bugs.webrtc.org/10662): Add stats for LNTF.
3327 
3328   info.timing_frame_info = stats.timing_frame_info;
3329 
3330   if (log_stats)
3331     RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
3332 
3333   return info;
3334 }
3335 
3336 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetRecordableEncodedFrameCallback(std::function<void (const webrtc::RecordableEncodedFrame &)> callback)3337     SetRecordableEncodedFrameCallback(
3338         std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
3339   if (stream_) {
3340     stream_->SetAndGetRecordingState(
3341         webrtc::VideoReceiveStreamInterface::RecordingState(
3342             std::move(callback)),
3343         /*generate_key_frame=*/true);
3344   } else {
3345     RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
3346                          "frame sink";
3347   }
3348 }
3349 
3350 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
ClearRecordableEncodedFrameCallback()3351     ClearRecordableEncodedFrameCallback() {
3352   if (stream_) {
3353     stream_->SetAndGetRecordingState(
3354         webrtc::VideoReceiveStreamInterface::RecordingState(),
3355         /*generate_key_frame=*/false);
3356   } else {
3357     RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
3358                          "frame sink";
3359   }
3360 }
3361 
GenerateKeyFrame()3362 void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() {
3363   if (stream_) {
3364     stream_->GenerateKeyFrame();
3365   } else {
3366     RTC_LOG(LS_ERROR)
3367         << "Absent receive stream; ignoring key frame generation request.";
3368   }
3369 }
3370 
3371 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3372     SetDepacketizerToDecoderFrameTransformer(
3373         rtc::scoped_refptr<webrtc::FrameTransformerInterface>
3374             frame_transformer) {
3375   config_.frame_transformer = frame_transformer;
3376   if (stream_)
3377     stream_->SetDepacketizerToDecoderFrameTransformer(frame_transformer);
3378 }
3379 
SetLocalSsrc(uint32_t ssrc)3380 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetLocalSsrc(uint32_t ssrc) {
3381   config_.rtp.local_ssrc = ssrc;
3382   call_->OnLocalSsrcUpdated(stream(), ssrc);
3383   if (flexfec_stream_)
3384     call_->OnLocalSsrcUpdated(*flexfec_stream_, ssrc);
3385 }
3386 
VideoCodecSettings()3387 WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings()
3388     : flexfec_payload_type(-1), rtx_payload_type(-1), rtx_time(-1) {}
3389 
operator ==(const WebRtcVideoChannel::VideoCodecSettings & other) const3390 bool WebRtcVideoChannel::VideoCodecSettings::operator==(
3391     const WebRtcVideoChannel::VideoCodecSettings& other) const {
3392   return codec == other.codec && ulpfec == other.ulpfec &&
3393          flexfec_payload_type == other.flexfec_payload_type &&
3394          rtx_payload_type == other.rtx_payload_type &&
3395          rtx_time == other.rtx_time;
3396 }
3397 
EqualsDisregardingFlexfec(const WebRtcVideoChannel::VideoCodecSettings & a,const WebRtcVideoChannel::VideoCodecSettings & b)3398 bool WebRtcVideoChannel::VideoCodecSettings::EqualsDisregardingFlexfec(
3399     const WebRtcVideoChannel::VideoCodecSettings& a,
3400     const WebRtcVideoChannel::VideoCodecSettings& b) {
3401   return a.codec == b.codec && a.ulpfec == b.ulpfec &&
3402          a.rtx_payload_type == b.rtx_payload_type && a.rtx_time == b.rtx_time;
3403 }
3404 
operator !=(const WebRtcVideoChannel::VideoCodecSettings & other) const3405 bool WebRtcVideoChannel::VideoCodecSettings::operator!=(
3406     const WebRtcVideoChannel::VideoCodecSettings& other) const {
3407   return !(*this == other);
3408 }
3409 
3410 std::vector<WebRtcVideoChannel::VideoCodecSettings>
MapCodecs(const std::vector<VideoCodec> & codecs)3411 WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) {
3412   if (codecs.empty()) {
3413     return {};
3414   }
3415 
3416   std::vector<VideoCodecSettings> video_codecs;
3417   std::map<int, VideoCodec::CodecType> payload_codec_type;
3418   // `rtx_mapping` maps video payload type to rtx payload type.
3419   std::map<int, int> rtx_mapping;
3420   std::map<int, int> rtx_time_mapping;
3421 
3422   webrtc::UlpfecConfig ulpfec_config;
3423   absl::optional<int> flexfec_payload_type;
3424 
3425   for (const VideoCodec& in_codec : codecs) {
3426     const int payload_type = in_codec.id;
3427 
3428     if (payload_codec_type.find(payload_type) != payload_codec_type.end()) {
3429       RTC_LOG(LS_ERROR) << "Payload type already registered: "
3430                         << in_codec.ToString();
3431       return {};
3432     }
3433     payload_codec_type[payload_type] = in_codec.GetCodecType();
3434 
3435     switch (in_codec.GetCodecType()) {
3436       case VideoCodec::CODEC_RED: {
3437         if (ulpfec_config.red_payload_type != -1) {
3438           RTC_LOG(LS_ERROR)
3439               << "Duplicate RED codec: ignoring PT=" << payload_type
3440               << " in favor of PT=" << ulpfec_config.red_payload_type
3441               << " which was specified first.";
3442           break;
3443         }
3444         ulpfec_config.red_payload_type = payload_type;
3445         break;
3446       }
3447 
3448       case VideoCodec::CODEC_ULPFEC: {
3449         if (ulpfec_config.ulpfec_payload_type != -1) {
3450           RTC_LOG(LS_ERROR)
3451               << "Duplicate ULPFEC codec: ignoring PT=" << payload_type
3452               << " in favor of PT=" << ulpfec_config.ulpfec_payload_type
3453               << " which was specified first.";
3454           break;
3455         }
3456         ulpfec_config.ulpfec_payload_type = payload_type;
3457         break;
3458       }
3459 
3460       case VideoCodec::CODEC_FLEXFEC: {
3461         if (flexfec_payload_type) {
3462           RTC_LOG(LS_ERROR)
3463               << "Duplicate FLEXFEC codec: ignoring PT=" << payload_type
3464               << " in favor of PT=" << *flexfec_payload_type
3465               << " which was specified first.";
3466           break;
3467         }
3468         flexfec_payload_type = payload_type;
3469         break;
3470       }
3471 
3472       case VideoCodec::CODEC_RTX: {
3473         int associated_payload_type;
3474         if (!in_codec.GetParam(kCodecParamAssociatedPayloadType,
3475                                &associated_payload_type) ||
3476             !IsValidRtpPayloadType(associated_payload_type)) {
3477           RTC_LOG(LS_ERROR)
3478               << "RTX codec with invalid or no associated payload type: "
3479               << in_codec.ToString();
3480           return {};
3481         }
3482         int rtx_time;
3483         if (in_codec.GetParam(kCodecParamRtxTime, &rtx_time) && rtx_time > 0) {
3484           rtx_time_mapping[associated_payload_type] = rtx_time;
3485         }
3486         rtx_mapping[associated_payload_type] = payload_type;
3487         break;
3488       }
3489 
3490       case VideoCodec::CODEC_VIDEO: {
3491         video_codecs.emplace_back();
3492         video_codecs.back().codec = in_codec;
3493         break;
3494       }
3495     }
3496   }
3497 
3498   // One of these codecs should have been a video codec. Only having FEC
3499   // parameters into this code is a logic error.
3500   RTC_DCHECK(!video_codecs.empty());
3501 
3502   for (const auto& entry : rtx_mapping) {
3503     const int associated_payload_type = entry.first;
3504     const int rtx_payload_type = entry.second;
3505     auto it = payload_codec_type.find(associated_payload_type);
3506     if (it == payload_codec_type.end()) {
3507       RTC_LOG(LS_ERROR) << "RTX codec (PT=" << rtx_payload_type
3508                         << ") mapped to PT=" << associated_payload_type
3509                         << " which is not in the codec list.";
3510       return {};
3511     }
3512     const VideoCodec::CodecType associated_codec_type = it->second;
3513     if (associated_codec_type != VideoCodec::CODEC_VIDEO &&
3514         associated_codec_type != VideoCodec::CODEC_RED) {
3515       RTC_LOG(LS_ERROR)
3516           << "RTX PT=" << rtx_payload_type
3517           << " not mapped to regular video codec or RED codec (PT="
3518           << associated_payload_type << ").";
3519       return {};
3520     }
3521 
3522     if (associated_payload_type == ulpfec_config.red_payload_type) {
3523       ulpfec_config.red_rtx_payload_type = rtx_payload_type;
3524     }
3525   }
3526 
3527   for (VideoCodecSettings& codec_settings : video_codecs) {
3528     const int payload_type = codec_settings.codec.id;
3529     codec_settings.ulpfec = ulpfec_config;
3530     codec_settings.flexfec_payload_type = flexfec_payload_type.value_or(-1);
3531     auto it = rtx_mapping.find(payload_type);
3532     if (it != rtx_mapping.end()) {
3533       const int rtx_payload_type = it->second;
3534       codec_settings.rtx_payload_type = rtx_payload_type;
3535 
3536       auto rtx_time_it = rtx_time_mapping.find(payload_type);
3537       if (rtx_time_it != rtx_time_mapping.end()) {
3538         const int rtx_time = rtx_time_it->second;
3539         if (rtx_time < kNackHistoryMs) {
3540           codec_settings.rtx_time = rtx_time;
3541         } else {
3542           codec_settings.rtx_time = kNackHistoryMs;
3543         }
3544       }
3545     }
3546   }
3547 
3548   return video_codecs;
3549 }
3550 
3551 WebRtcVideoChannel::WebRtcVideoReceiveStream*
FindReceiveStream(uint32_t ssrc)3552 WebRtcVideoChannel::FindReceiveStream(uint32_t ssrc) {
3553   if (ssrc == 0) {
3554     absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
3555     if (!default_ssrc) {
3556       return nullptr;
3557     }
3558     ssrc = *default_ssrc;
3559   }
3560   auto it = receive_streams_.find(ssrc);
3561   if (it != receive_streams_.end()) {
3562     return it->second;
3563   }
3564   return nullptr;
3565 }
3566 
SetRecordableEncodedFrameCallback(uint32_t ssrc,std::function<void (const webrtc::RecordableEncodedFrame &)> callback)3567 void WebRtcVideoChannel::SetRecordableEncodedFrameCallback(
3568     uint32_t ssrc,
3569     std::function<void(const webrtc::RecordableEncodedFrame&)> callback) {
3570   RTC_DCHECK_RUN_ON(&thread_checker_);
3571   WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3572   if (stream) {
3573     stream->SetRecordableEncodedFrameCallback(std::move(callback));
3574   } else {
3575     RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring setting encoded "
3576                          "frame sink for ssrc "
3577                       << ssrc;
3578   }
3579 }
3580 
ClearRecordableEncodedFrameCallback(uint32_t ssrc)3581 void WebRtcVideoChannel::ClearRecordableEncodedFrameCallback(uint32_t ssrc) {
3582   RTC_DCHECK_RUN_ON(&thread_checker_);
3583   WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3584   if (stream) {
3585     stream->ClearRecordableEncodedFrameCallback();
3586   } else {
3587     RTC_LOG(LS_ERROR) << "Absent receive stream; ignoring clearing encoded "
3588                          "frame sink for ssrc "
3589                       << ssrc;
3590   }
3591 }
3592 
RequestRecvKeyFrame(uint32_t ssrc)3593 void WebRtcVideoChannel::RequestRecvKeyFrame(uint32_t ssrc) {
3594   RTC_DCHECK_RUN_ON(&thread_checker_);
3595   WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc);
3596   if (stream) {
3597     return stream->GenerateKeyFrame();
3598   } else {
3599     RTC_LOG(LS_ERROR)
3600         << "Absent receive stream; ignoring key frame generation for ssrc "
3601         << ssrc;
3602   }
3603 }
3604 
GenerateSendKeyFrame(uint32_t ssrc,const std::vector<std::string> & rids)3605 void WebRtcVideoChannel::GenerateSendKeyFrame(
3606     uint32_t ssrc,
3607     const std::vector<std::string>& rids) {
3608   RTC_DCHECK_RUN_ON(&thread_checker_);
3609   auto it = send_streams_.find(ssrc);
3610   if (it != send_streams_.end()) {
3611     it->second->GenerateKeyFrame(rids);
3612   } else {
3613     RTC_LOG(LS_ERROR)
3614         << "Absent send stream; ignoring key frame generation for ssrc "
3615         << ssrc;
3616   }
3617 }
3618 
SetEncoderToPacketizerFrameTransformer(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3619 void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
3620     uint32_t ssrc,
3621     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
3622   RTC_DCHECK_RUN_ON(&thread_checker_);
3623   auto matching_stream = send_streams_.find(ssrc);
3624   if (matching_stream != send_streams_.end()) {
3625     matching_stream->second->SetEncoderToPacketizerFrameTransformer(
3626         std::move(frame_transformer));
3627   }
3628 }
3629 
SetDepacketizerToDecoderFrameTransformer(uint32_t ssrc,rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)3630 void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer(
3631     uint32_t ssrc,
3632     rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
3633   RTC_DCHECK(frame_transformer);
3634   RTC_DCHECK_RUN_ON(&thread_checker_);
3635   if (ssrc == 0) {
3636     // If the receiver is unsignaled, save the frame transformer and set it when
3637     // the stream is associated with an ssrc.
3638     unsignaled_frame_transformer_ = std::move(frame_transformer);
3639     return;
3640   }
3641 
3642   auto matching_stream = receive_streams_.find(ssrc);
3643   if (matching_stream != receive_streams_.end()) {
3644     matching_stream->second->SetDepacketizerToDecoderFrameTransformer(
3645         std::move(frame_transformer));
3646   }
3647 }
3648 
3649 }  // namespace cricket
3650