1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_VIDEO_SEND_STREAM_H_ 12 #define CALL_VIDEO_SEND_STREAM_H_ 13 14 #include <stdint.h> 15 16 #include <map> 17 #include <string> 18 #include <vector> 19 20 #include "absl/types/optional.h" 21 #include "api/adaptation/resource.h" 22 #include "api/call/transport.h" 23 #include "api/crypto/crypto_options.h" 24 #include "api/frame_transformer_interface.h" 25 #include "api/rtp_parameters.h" 26 #include "api/rtp_sender_interface.h" 27 #include "api/scoped_refptr.h" 28 #include "api/video/video_content_type.h" 29 #include "api/video/video_frame.h" 30 #include "api/video/video_sink_interface.h" 31 #include "api/video/video_source_interface.h" 32 #include "api/video/video_stream_encoder_settings.h" 33 #include "call/rtp_config.h" 34 #include "common_video/frame_counts.h" 35 #include "common_video/include/quality_limitation_reason.h" 36 #include "modules/rtp_rtcp/include/report_block_data.h" 37 #include "modules/rtp_rtcp/include/rtcp_statistics.h" 38 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 39 #include "video/config/video_encoder_config.h" 40 41 namespace webrtc { 42 43 class FrameEncryptorInterface; 44 45 class VideoSendStream { 46 public: 47 // Multiple StreamStats objects are present if simulcast is used (multiple 48 // kMedia streams) or if RTX or FlexFEC is negotiated. Multiple SVC layers, on 49 // the other hand, does not cause additional StreamStats. 50 struct StreamStats { 51 enum class StreamType { 52 // A media stream is an RTP stream for audio or video. Retransmissions and 53 // FEC is either sent over the same SSRC or negotiated to be sent over 54 // separate SSRCs, in which case separate StreamStats objects exist with 55 // references to this media stream's SSRC. 56 kMedia, 57 // RTX streams are streams dedicated to retransmissions. They have a 58 // dependency on a single kMedia stream: `referenced_media_ssrc`. 59 kRtx, 60 // FlexFEC streams are streams dedicated to FlexFEC. They have a 61 // dependency on a single kMedia stream: `referenced_media_ssrc`. 62 kFlexfec, 63 }; 64 65 StreamStats(); 66 ~StreamStats(); 67 68 std::string ToString() const; 69 70 StreamType type = StreamType::kMedia; 71 // If `type` is kRtx or kFlexfec this value is present. The referenced SSRC 72 // is the kMedia stream that this stream is performing retransmissions or 73 // FEC for. If `type` is kMedia, this value is null. 74 absl::optional<uint32_t> referenced_media_ssrc; 75 FrameCounts frame_counts; 76 int width = 0; 77 int height = 0; 78 // TODO(holmer): Move bitrate_bps out to the webrtc::Call layer. 79 int total_bitrate_bps = 0; 80 int retransmit_bitrate_bps = 0; 81 // `avg_delay_ms` and `max_delay_ms` are only used in tests. Consider 82 // deleting. 83 int avg_delay_ms = 0; 84 int max_delay_ms = 0; 85 StreamDataCounters rtp_stats; 86 RtcpPacketTypeCounter rtcp_packet_type_counts; 87 // A snapshot of the most recent Report Block with additional data of 88 // interest to statistics. Used to implement RTCRemoteInboundRtpStreamStats. 89 absl::optional<ReportBlockData> report_block_data; 90 double encode_frame_rate = 0.0; 91 int frames_encoded = 0; 92 absl::optional<uint64_t> qp_sum; 93 uint64_t total_encode_time_ms = 0; 94 uint64_t total_encoded_bytes_target = 0; 95 uint32_t huge_frames_sent = 0; 96 }; 97 98 struct Stats { 99 Stats(); 100 ~Stats(); 101 std::string ToString(int64_t time_ms) const; 102 std::string encoder_implementation_name = "unknown"; 103 double input_frame_rate = 0; 104 int encode_frame_rate = 0; 105 int avg_encode_time_ms = 0; 106 int encode_usage_percent = 0; 107 uint32_t frames_encoded = 0; 108 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime 109 uint64_t total_encode_time_ms = 0; 110 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget 111 uint64_t total_encoded_bytes_target = 0; 112 uint32_t frames = 0; 113 uint32_t frames_dropped_by_capturer = 0; 114 uint32_t frames_dropped_by_encoder_queue = 0; 115 uint32_t frames_dropped_by_rate_limiter = 0; 116 uint32_t frames_dropped_by_congestion_window = 0; 117 uint32_t frames_dropped_by_encoder = 0; 118 // Bitrate the encoder is currently configured to use due to bandwidth 119 // limitations. 120 int target_media_bitrate_bps = 0; 121 // Bitrate the encoder is actually producing. 122 int media_bitrate_bps = 0; 123 bool suspended = false; 124 bool bw_limited_resolution = false; 125 bool cpu_limited_resolution = false; 126 bool bw_limited_framerate = false; 127 bool cpu_limited_framerate = false; 128 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason 129 QualityLimitationReason quality_limitation_reason = 130 QualityLimitationReason::kNone; 131 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations 132 std::map<QualityLimitationReason, int64_t> quality_limitation_durations_ms; 133 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges 134 uint32_t quality_limitation_resolution_changes = 0; 135 // Total number of times resolution as been requested to be changed due to 136 // CPU/quality adaptation. 137 int number_of_cpu_adapt_changes = 0; 138 int number_of_quality_adapt_changes = 0; 139 bool has_entered_low_resolution = false; 140 std::map<uint32_t, StreamStats> substreams; 141 webrtc::VideoContentType content_type = 142 webrtc::VideoContentType::UNSPECIFIED; 143 uint32_t frames_sent = 0; 144 uint32_t huge_frames_sent = 0; 145 absl::optional<bool> power_efficient_encoder; 146 }; 147 148 struct Config { 149 public: 150 Config() = delete; 151 Config(Config&&); 152 explicit Config(Transport* send_transport); 153 154 Config& operator=(Config&&); 155 Config& operator=(const Config&) = delete; 156 157 ~Config(); 158 159 // Mostly used by tests. Avoid creating copies if you can. CopyConfig160 Config Copy() const { return Config(*this); } 161 162 std::string ToString() const; 163 164 RtpConfig rtp; 165 166 VideoStreamEncoderSettings encoder_settings; 167 168 // Time interval between RTCP report for video 169 int rtcp_report_interval_ms = 1000; 170 171 // Transport for outgoing packets. 172 Transport* send_transport = nullptr; 173 174 // Expected delay needed by the renderer, i.e. the frame will be delivered 175 // this many milliseconds, if possible, earlier than expected render time. 176 // Only valid if `local_renderer` is set. 177 int render_delay_ms = 0; 178 179 // Target delay in milliseconds. A positive value indicates this stream is 180 // used for streaming instead of a real-time call. 181 int target_delay_ms = 0; 182 183 // True if the stream should be suspended when the available bitrate fall 184 // below the minimum configured bitrate. If this variable is false, the 185 // stream may send at a rate higher than the estimated available bitrate. 186 bool suspend_below_min_bitrate = false; 187 188 // Enables periodic bandwidth probing in application-limited region. 189 bool periodic_alr_bandwidth_probing = false; 190 191 // An optional custom frame encryptor that allows the entire frame to be 192 // encrypted in whatever way the caller chooses. This is not required by 193 // default. 194 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor; 195 196 // An optional encoder selector provided by the user. 197 // Overrides VideoEncoderFactory::GetEncoderSelector(). 198 // Owned by RtpSenderBase. 199 VideoEncoderFactory::EncoderSelectorInterface* encoder_selector = nullptr; 200 201 // Per PeerConnection cryptography options. 202 CryptoOptions crypto_options; 203 204 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer; 205 206 private: 207 // Access to the copy constructor is private to force use of the Copy() 208 // method for those exceptional cases where we do use it. 209 Config(const Config&); 210 }; 211 212 // Updates the sending state for all simulcast layers that the video send 213 // stream owns. This can mean updating the activity one or for multiple 214 // layers. The ordering of active layers is the order in which the 215 // rtp modules are stored in the VideoSendStream. 216 // Note: This starts stream activity if it is inactive and one of the layers 217 // is active. This stops stream activity if it is active and all layers are 218 // inactive. 219 // `active_layers` should have the same size as the number of configured 220 // simulcast layers or one if only one rtp stream is used. 221 virtual void StartPerRtpStream(std::vector<bool> active_layers) = 0; 222 223 // Starts stream activity. 224 // When a stream is active, it can receive, process and deliver packets. 225 // Prefer to use StartPerRtpStream. 226 virtual void Start() = 0; 227 228 // Stops stream activity. 229 // When a stream is stopped, it can't receive, process or deliver packets. 230 virtual void Stop() = 0; 231 232 // Accessor for determining if the stream is active. This is an inexpensive 233 // call that must be made on the same thread as `Start()` and `Stop()` methods 234 // are called on and will return `true` iff activity has been started either 235 // via `Start()` or `StartPerRtpStream()`. If activity is either 236 // stopped or is in the process of being stopped as a result of a call to 237 // either `Stop()` or `StartPerRtpStream()` where all layers were 238 // deactivated, the return value will be `false`. 239 virtual bool started() = 0; 240 241 // If the resource is overusing, the VideoSendStream will try to reduce 242 // resolution or frame rate until no resource is overusing. 243 // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor 244 // is moved to Call this method could be deleted altogether in favor of 245 // Call-level APIs only. 246 virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0; 247 virtual std::vector<rtc::scoped_refptr<Resource>> 248 GetAdaptationResources() = 0; 249 250 virtual void SetSource( 251 rtc::VideoSourceInterface<webrtc::VideoFrame>* source, 252 const DegradationPreference& degradation_preference) = 0; 253 254 // Set which streams to send. Must have at least as many SSRCs as configured 255 // in the config. Encoder settings are passed on to the encoder instance along 256 // with the VideoStream settings. 257 virtual void ReconfigureVideoEncoder(VideoEncoderConfig config) = 0; 258 259 virtual void ReconfigureVideoEncoder(VideoEncoderConfig config, 260 SetParametersCallback callback) = 0; 261 262 virtual Stats GetStats() = 0; 263 264 virtual void GenerateKeyFrame(const std::vector<std::string>& rids) = 0; 265 266 protected: ~VideoSendStream()267 virtual ~VideoSendStream() {} 268 }; 269 270 } // namespace webrtc 271 272 #endif // CALL_VIDEO_SEND_STREAM_H_ 273