1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11
12 #include <algorithm> // std::max
13
14 #include "webrtc/base/checks.h"
15 #include "webrtc/base/logging.h"
16 #include "webrtc/common_types.h"
17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
18 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
19 #include "webrtc/modules/video_coding/encoded_frame.h"
20 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
21 #include "webrtc/modules/video_coding/video_coding_impl.h"
22 #include "webrtc/system_wrappers/include/clock.h"
23
24 namespace webrtc {
25 namespace vcm {
26
VideoSender(Clock * clock,EncodedImageCallback * post_encode_callback,VideoEncoderRateObserver * encoder_rate_observer,VCMQMSettingsCallback * qm_settings_callback)27 VideoSender::VideoSender(Clock* clock,
28 EncodedImageCallback* post_encode_callback,
29 VideoEncoderRateObserver* encoder_rate_observer,
30 VCMQMSettingsCallback* qm_settings_callback)
31 : clock_(clock),
32 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
33 _encoder(nullptr),
34 _encodedFrameCallback(post_encode_callback),
35 _nextFrameTypes(1, kVideoFrameDelta),
36 _mediaOpt(clock_),
37 _sendStatsCallback(nullptr),
38 _codecDataBase(encoder_rate_observer, &_encodedFrameCallback),
39 frame_dropper_enabled_(true),
40 _sendStatsTimer(1000, clock_),
41 current_codec_(),
42 qm_settings_callback_(qm_settings_callback),
43 protection_callback_(nullptr),
44 encoder_params_({0, 0, 0, 0}) {
45 // Allow VideoSender to be created on one thread but used on another, post
46 // construction. This is currently how this class is being used by at least
47 // one external project (diffractor).
48 _mediaOpt.EnableQM(qm_settings_callback_ != nullptr);
49 _mediaOpt.Reset();
50 main_thread_.DetachFromThread();
51 }
52
~VideoSender()53 VideoSender::~VideoSender() {}
54
Process()55 int32_t VideoSender::Process() {
56 int32_t returnValue = VCM_OK;
57
58 if (_sendStatsTimer.TimeUntilProcess() == 0) {
59 _sendStatsTimer.Processed();
60 CriticalSectionScoped cs(process_crit_sect_.get());
61 if (_sendStatsCallback != nullptr) {
62 uint32_t bitRate = _mediaOpt.SentBitRate();
63 uint32_t frameRate = _mediaOpt.SentFrameRate();
64 _sendStatsCallback->SendStatistics(bitRate, frameRate);
65 }
66 }
67
68 {
69 rtc::CritScope cs(¶ms_lock_);
70 // Force an encoder parameters update, so that incoming frame rate is
71 // updated even if bandwidth hasn't changed.
72 encoder_params_.input_frame_rate = _mediaOpt.InputFrameRate();
73 }
74
75 return returnValue;
76 }
77
TimeUntilNextProcess()78 int64_t VideoSender::TimeUntilNextProcess() {
79 return _sendStatsTimer.TimeUntilProcess();
80 }
81
82 // Register the send codec to be used.
RegisterSendCodec(const VideoCodec * sendCodec,uint32_t numberOfCores,uint32_t maxPayloadSize)83 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
84 uint32_t numberOfCores,
85 uint32_t maxPayloadSize) {
86 RTC_DCHECK(main_thread_.CalledOnValidThread());
87 rtc::CritScope lock(&send_crit_);
88 if (sendCodec == nullptr) {
89 return VCM_PARAMETER_ERROR;
90 }
91
92 bool ret =
93 _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize);
94
95 // Update encoder regardless of result to make sure that we're not holding on
96 // to a deleted instance.
97 _encoder = _codecDataBase.GetEncoder();
98 // Cache the current codec here so they can be fetched from this thread
99 // without requiring the _sendCritSect lock.
100 current_codec_ = *sendCodec;
101
102 if (!ret) {
103 LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '"
104 << sendCodec->plName << "'.";
105 return VCM_CODEC_ERROR;
106 }
107
108 int numLayers;
109 if (sendCodec->codecType == kVideoCodecVP8) {
110 numLayers = sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
111 } else if (sendCodec->codecType == kVideoCodecVP9) {
112 numLayers = sendCodec->codecSpecific.VP9.numberOfTemporalLayers;
113 } else {
114 numLayers = 1;
115 }
116
117 // If we have screensharing and we have layers, we disable frame dropper.
118 bool disable_frame_dropper =
119 numLayers > 1 && sendCodec->mode == kScreensharing;
120 if (disable_frame_dropper) {
121 _mediaOpt.EnableFrameDropper(false);
122 } else if (frame_dropper_enabled_) {
123 _mediaOpt.EnableFrameDropper(true);
124 }
125 _nextFrameTypes.clear();
126 _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
127 kVideoFrameDelta);
128
129 _mediaOpt.SetEncodingData(sendCodec->codecType, sendCodec->maxBitrate * 1000,
130 sendCodec->startBitrate * 1000, sendCodec->width,
131 sendCodec->height, sendCodec->maxFramerate,
132 numLayers, maxPayloadSize);
133 return VCM_OK;
134 }
135
136 // Register an external decoder object.
137 // This can not be used together with external decoder callbacks.
RegisterExternalEncoder(VideoEncoder * externalEncoder,uint8_t payloadType,bool internalSource)138 void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
139 uint8_t payloadType,
140 bool internalSource /*= false*/) {
141 RTC_DCHECK(main_thread_.CalledOnValidThread());
142
143 rtc::CritScope lock(&send_crit_);
144
145 if (externalEncoder == nullptr) {
146 bool wasSendCodec = false;
147 RTC_CHECK(
148 _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec));
149 if (wasSendCodec) {
150 // Make sure the VCM doesn't use the de-registered codec
151 _encoder = nullptr;
152 }
153 return;
154 }
155 _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType,
156 internalSource);
157 }
158
159 // Get encode bitrate
Bitrate(unsigned int * bitrate) const160 int VideoSender::Bitrate(unsigned int* bitrate) const {
161 RTC_DCHECK(main_thread_.CalledOnValidThread());
162 // Since we're running on the thread that's the only thread known to modify
163 // the value of _encoder, we don't need to grab the lock here.
164
165 if (!_encoder)
166 return VCM_UNINITIALIZED;
167 *bitrate = _encoder->GetEncoderParameters().target_bitrate;
168 return 0;
169 }
170
171 // Get encode frame rate
FrameRate(unsigned int * framerate) const172 int VideoSender::FrameRate(unsigned int* framerate) const {
173 RTC_DCHECK(main_thread_.CalledOnValidThread());
174 // Since we're running on the thread that's the only thread known to modify
175 // the value of _encoder, we don't need to grab the lock here.
176
177 if (!_encoder)
178 return VCM_UNINITIALIZED;
179
180 *framerate = _encoder->GetEncoderParameters().input_frame_rate;
181 return 0;
182 }
183
SetChannelParameters(uint32_t target_bitrate,uint8_t lossRate,int64_t rtt)184 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
185 uint8_t lossRate,
186 int64_t rtt) {
187 uint32_t target_rate =
188 _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt,
189 protection_callback_, qm_settings_callback_);
190
191 uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
192
193 rtc::CritScope cs(¶ms_lock_);
194 encoder_params_ = {target_rate, lossRate, rtt, input_frame_rate};
195
196 return VCM_OK;
197 }
198
SetEncoderParameters(EncoderParameters params)199 void VideoSender::SetEncoderParameters(EncoderParameters params) {
200 if (params.target_bitrate == 0)
201 return;
202
203 if (params.input_frame_rate == 0) {
204 // No frame rate estimate available, use default.
205 params.input_frame_rate = current_codec_.maxFramerate;
206 }
207 if (_encoder != nullptr)
208 _encoder->SetEncoderParameters(params);
209 }
210
RegisterTransportCallback(VCMPacketizationCallback * transport)211 int32_t VideoSender::RegisterTransportCallback(
212 VCMPacketizationCallback* transport) {
213 rtc::CritScope lock(&send_crit_);
214 _encodedFrameCallback.SetMediaOpt(&_mediaOpt);
215 _encodedFrameCallback.SetTransportCallback(transport);
216 return VCM_OK;
217 }
218
219 // Register video output information callback which will be called to deliver
220 // information about the video stream produced by the encoder, for instance the
221 // average frame rate and bit rate.
RegisterSendStatisticsCallback(VCMSendStatisticsCallback * sendStats)222 int32_t VideoSender::RegisterSendStatisticsCallback(
223 VCMSendStatisticsCallback* sendStats) {
224 CriticalSectionScoped cs(process_crit_sect_.get());
225 _sendStatsCallback = sendStats;
226 return VCM_OK;
227 }
228
229 // Register a video protection callback which will be called to deliver the
230 // requested FEC rate and NACK status (on/off).
231 // Note: this callback is assumed to only be registered once and before it is
232 // used in this class.
RegisterProtectionCallback(VCMProtectionCallback * protection_callback)233 int32_t VideoSender::RegisterProtectionCallback(
234 VCMProtectionCallback* protection_callback) {
235 RTC_DCHECK(protection_callback == nullptr || protection_callback_ == nullptr);
236 protection_callback_ = protection_callback;
237 return VCM_OK;
238 }
239
240 // Enable or disable a video protection method.
SetVideoProtection(VCMVideoProtection videoProtection)241 void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) {
242 rtc::CritScope lock(&send_crit_);
243 switch (videoProtection) {
244 case kProtectionNone:
245 _mediaOpt.SetProtectionMethod(media_optimization::kNone);
246 break;
247 case kProtectionNack:
248 _mediaOpt.SetProtectionMethod(media_optimization::kNack);
249 break;
250 case kProtectionNackFEC:
251 _mediaOpt.SetProtectionMethod(media_optimization::kNackFec);
252 break;
253 case kProtectionFEC:
254 _mediaOpt.SetProtectionMethod(media_optimization::kFec);
255 break;
256 }
257 }
258 // Add one raw video frame to the encoder, blocking.
AddVideoFrame(const VideoFrame & videoFrame,const VideoContentMetrics * contentMetrics,const CodecSpecificInfo * codecSpecificInfo)259 int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
260 const VideoContentMetrics* contentMetrics,
261 const CodecSpecificInfo* codecSpecificInfo) {
262 EncoderParameters encoder_params;
263 {
264 rtc::CritScope lock(¶ms_lock_);
265 encoder_params = encoder_params_;
266 }
267 rtc::CritScope lock(&send_crit_);
268 if (_encoder == nullptr)
269 return VCM_UNINITIALIZED;
270 SetEncoderParameters(encoder_params);
271 // TODO(holmer): Add support for dropping frames per stream. Currently we
272 // only have one frame dropper for all streams.
273 if (_nextFrameTypes[0] == kEmptyFrame) {
274 return VCM_OK;
275 }
276 if (_mediaOpt.DropFrame()) {
277 _encoder->OnDroppedFrame();
278 return VCM_OK;
279 }
280 _mediaOpt.UpdateContentData(contentMetrics);
281 // TODO(pbos): Make sure setting send codec is synchronized with video
282 // processing so frame size always matches.
283 if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
284 videoFrame.height())) {
285 LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping.";
286 return VCM_PARAMETER_ERROR;
287 }
288 VideoFrame converted_frame = videoFrame;
289 if (converted_frame.native_handle() && !_encoder->SupportsNativeHandle()) {
290 // This module only supports software encoding.
291 // TODO(pbos): Offload conversion from the encoder thread.
292 converted_frame = converted_frame.ConvertNativeToI420Frame();
293 RTC_CHECK(!converted_frame.IsZeroSize())
294 << "Frame conversion failed, won't be able to encode frame.";
295 }
296 int32_t ret =
297 _encoder->Encode(converted_frame, codecSpecificInfo, _nextFrameTypes);
298 if (ret < 0) {
299 LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
300 return ret;
301 }
302 for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
303 _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type.
304 }
305 if (qm_settings_callback_)
306 qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate());
307 return VCM_OK;
308 }
309
IntraFrameRequest(int stream_index)310 int32_t VideoSender::IntraFrameRequest(int stream_index) {
311 rtc::CritScope lock(&send_crit_);
312 if (stream_index < 0 ||
313 static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) {
314 return -1;
315 }
316 _nextFrameTypes[stream_index] = kVideoFrameKey;
317 if (_encoder != nullptr && _encoder->InternalSource()) {
318 // Try to request the frame if we have an external encoder with
319 // internal source since AddVideoFrame never will be called.
320 if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) {
321 _nextFrameTypes[stream_index] = kVideoFrameDelta;
322 }
323 }
324 return VCM_OK;
325 }
326
EnableFrameDropper(bool enable)327 int32_t VideoSender::EnableFrameDropper(bool enable) {
328 rtc::CritScope lock(&send_crit_);
329 frame_dropper_enabled_ = enable;
330 _mediaOpt.EnableFrameDropper(enable);
331 return VCM_OK;
332 }
333
SuspendBelowMinBitrate()334 void VideoSender::SuspendBelowMinBitrate() {
335 RTC_DCHECK(main_thread_.CalledOnValidThread());
336 int threshold_bps;
337 if (current_codec_.numberOfSimulcastStreams == 0) {
338 threshold_bps = current_codec_.minBitrate * 1000;
339 } else {
340 threshold_bps = current_codec_.simulcastStream[0].minBitrate * 1000;
341 }
342 // Set the hysteresis window to be at 10% of the threshold, but at least
343 // 10 kbps.
344 int window_bps = std::max(threshold_bps / 10, 10000);
345 _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps);
346 }
347
VideoSuspended() const348 bool VideoSender::VideoSuspended() const {
349 return _mediaOpt.IsVideoSuspended();
350 }
351 } // namespace vcm
352 } // namespace webrtc
353