1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/common_types.h"
12
13 #include <algorithm> // std::max
14
15 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
18 #include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
19 #include "webrtc/system_wrappers/interface/clock.h"
20 #include "webrtc/system_wrappers/interface/logging.h"
21
22 namespace webrtc {
23 namespace vcm {
24
25 class DebugRecorder {
26 public:
DebugRecorder()27 DebugRecorder()
28 : cs_(CriticalSectionWrapper::CreateCriticalSection()), file_(NULL) {}
29
~DebugRecorder()30 ~DebugRecorder() { Stop(); }
31
Start(const char * file_name_utf8)32 int Start(const char* file_name_utf8) {
33 CriticalSectionScoped cs(cs_.get());
34 if (file_)
35 fclose(file_);
36 file_ = fopen(file_name_utf8, "wb");
37 if (!file_)
38 return VCM_GENERAL_ERROR;
39 return VCM_OK;
40 }
41
Stop()42 void Stop() {
43 CriticalSectionScoped cs(cs_.get());
44 if (file_) {
45 fclose(file_);
46 file_ = NULL;
47 }
48 }
49
Add(const I420VideoFrame & frame)50 void Add(const I420VideoFrame& frame) {
51 CriticalSectionScoped cs(cs_.get());
52 if (file_)
53 PrintI420VideoFrame(frame, file_);
54 }
55
56 private:
57 scoped_ptr<CriticalSectionWrapper> cs_;
58 FILE* file_ GUARDED_BY(cs_);
59 };
60
VideoSender(Clock * clock,EncodedImageCallback * post_encode_callback)61 VideoSender::VideoSender(Clock* clock,
62 EncodedImageCallback* post_encode_callback)
63 : clock_(clock),
64 recorder_(new DebugRecorder()),
65 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
66 _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()),
67 _encoder(),
68 _encodedFrameCallback(post_encode_callback),
69 _nextFrameTypes(1, kVideoFrameDelta),
70 _mediaOpt(clock_),
71 _sendStatsCallback(NULL),
72 _codecDataBase(),
73 frame_dropper_enabled_(true),
74 _sendStatsTimer(1000, clock_),
75 qm_settings_callback_(NULL),
76 protection_callback_(NULL) {}
77
~VideoSender()78 VideoSender::~VideoSender() {
79 delete _sendCritSect;
80 }
81
Process()82 int32_t VideoSender::Process() {
83 int32_t returnValue = VCM_OK;
84
85 if (_sendStatsTimer.TimeUntilProcess() == 0) {
86 _sendStatsTimer.Processed();
87 CriticalSectionScoped cs(process_crit_sect_.get());
88 if (_sendStatsCallback != NULL) {
89 uint32_t bitRate;
90 uint32_t frameRate;
91 {
92 CriticalSectionScoped cs(_sendCritSect);
93 bitRate = _mediaOpt.SentBitRate();
94 frameRate = _mediaOpt.SentFrameRate();
95 }
96 _sendStatsCallback->SendStatistics(bitRate, frameRate);
97 }
98 }
99
100 return returnValue;
101 }
102
103 // Reset send side to initial state - all components
InitializeSender()104 int32_t VideoSender::InitializeSender() {
105 CriticalSectionScoped cs(_sendCritSect);
106 _codecDataBase.ResetSender();
107 _encoder = NULL;
108 _encodedFrameCallback.SetTransportCallback(NULL);
109 _mediaOpt.Reset(); // Resetting frame dropper
110 return VCM_OK;
111 }
112
TimeUntilNextProcess()113 int32_t VideoSender::TimeUntilNextProcess() {
114 return _sendStatsTimer.TimeUntilProcess();
115 }
116
117 // Register the send codec to be used.
RegisterSendCodec(const VideoCodec * sendCodec,uint32_t numberOfCores,uint32_t maxPayloadSize)118 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
119 uint32_t numberOfCores,
120 uint32_t maxPayloadSize) {
121 CriticalSectionScoped cs(_sendCritSect);
122 if (sendCodec == NULL) {
123 return VCM_PARAMETER_ERROR;
124 }
125
126 bool ret = _codecDataBase.SetSendCodec(
127 sendCodec, numberOfCores, maxPayloadSize, &_encodedFrameCallback);
128
129 // Update encoder regardless of result to make sure that we're not holding on
130 // to a deleted instance.
131 _encoder = _codecDataBase.GetEncoder();
132
133 if (!ret) {
134 LOG(LS_ERROR) << "Failed to initialize the encoder with payload name "
135 << sendCodec->plName << ". Error code: " << ret;
136 return VCM_CODEC_ERROR;
137 }
138
139 int numLayers = (sendCodec->codecType != kVideoCodecVP8)
140 ? 1
141 : sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
142 // If we have screensharing and we have layers, we disable frame dropper.
143 bool disable_frame_dropper =
144 numLayers > 1 && sendCodec->mode == kScreensharing;
145 if (disable_frame_dropper) {
146 _mediaOpt.EnableFrameDropper(false);
147 } else if (frame_dropper_enabled_) {
148 _mediaOpt.EnableFrameDropper(true);
149 }
150 _nextFrameTypes.clear();
151 _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
152 kVideoFrameDelta);
153
154 _mediaOpt.SetEncodingData(sendCodec->codecType,
155 sendCodec->maxBitrate * 1000,
156 sendCodec->maxFramerate * 1000,
157 sendCodec->startBitrate * 1000,
158 sendCodec->width,
159 sendCodec->height,
160 numLayers,
161 maxPayloadSize);
162 return VCM_OK;
163 }
164
165 // Get current send codec
SendCodec(VideoCodec * currentSendCodec) const166 int32_t VideoSender::SendCodec(VideoCodec* currentSendCodec) const {
167 CriticalSectionScoped cs(_sendCritSect);
168
169 if (currentSendCodec == NULL) {
170 return VCM_PARAMETER_ERROR;
171 }
172 return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1;
173 }
174
175 // Get the current send codec type
SendCodec() const176 VideoCodecType VideoSender::SendCodec() const {
177 CriticalSectionScoped cs(_sendCritSect);
178
179 return _codecDataBase.SendCodec();
180 }
181
182 // Register an external decoder object.
183 // This can not be used together with external decoder callbacks.
RegisterExternalEncoder(VideoEncoder * externalEncoder,uint8_t payloadType,bool internalSource)184 int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
185 uint8_t payloadType,
186 bool internalSource /*= false*/) {
187 CriticalSectionScoped cs(_sendCritSect);
188
189 if (externalEncoder == NULL) {
190 bool wasSendCodec = false;
191 const bool ret =
192 _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec);
193 if (wasSendCodec) {
194 // Make sure the VCM doesn't use the de-registered codec
195 _encoder = NULL;
196 }
197 return ret ? 0 : -1;
198 }
199 _codecDataBase.RegisterExternalEncoder(
200 externalEncoder, payloadType, internalSource);
201 return 0;
202 }
203
204 // Get codec config parameters
CodecConfigParameters(uint8_t * buffer,int32_t size) const205 int32_t VideoSender::CodecConfigParameters(uint8_t* buffer,
206 int32_t size) const {
207 CriticalSectionScoped cs(_sendCritSect);
208 if (_encoder != NULL) {
209 return _encoder->CodecConfigParameters(buffer, size);
210 }
211 return VCM_UNINITIALIZED;
212 }
213
214 // TODO(andresp): Make const once media_opt is thread-safe and this has a
215 // pointer to it.
SentFrameCount(VCMFrameCount * frameCount)216 int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) {
217 CriticalSectionScoped cs(_sendCritSect);
218 *frameCount = _mediaOpt.SentFrameCount();
219 return VCM_OK;
220 }
221
222 // Get encode bitrate
Bitrate(unsigned int * bitrate) const223 int VideoSender::Bitrate(unsigned int* bitrate) const {
224 CriticalSectionScoped cs(_sendCritSect);
225 // return the bit rate which the encoder is set to
226 if (!_encoder) {
227 return VCM_UNINITIALIZED;
228 }
229 *bitrate = _encoder->BitRate();
230 return 0;
231 }
232
233 // Get encode frame rate
FrameRate(unsigned int * framerate) const234 int VideoSender::FrameRate(unsigned int* framerate) const {
235 CriticalSectionScoped cs(_sendCritSect);
236 // input frame rate, not compensated
237 if (!_encoder) {
238 return VCM_UNINITIALIZED;
239 }
240 *framerate = _encoder->FrameRate();
241 return 0;
242 }
243
244 // Set channel parameters
SetChannelParameters(uint32_t target_bitrate,uint8_t lossRate,uint32_t rtt)245 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
246 uint8_t lossRate,
247 uint32_t rtt) {
248 int32_t ret = 0;
249 {
250 CriticalSectionScoped sendCs(_sendCritSect);
251 uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate,
252 lossRate,
253 rtt,
254 protection_callback_,
255 qm_settings_callback_);
256 if (_encoder != NULL) {
257 ret = _encoder->SetChannelParameters(lossRate, rtt);
258 if (ret < 0) {
259 return ret;
260 }
261 ret = (int32_t)_encoder->SetRates(targetRate, _mediaOpt.InputFrameRate());
262 if (ret < 0) {
263 return ret;
264 }
265 } else {
266 return VCM_UNINITIALIZED;
267 } // encoder
268 } // send side
269 return VCM_OK;
270 }
271
RegisterTransportCallback(VCMPacketizationCallback * transport)272 int32_t VideoSender::RegisterTransportCallback(
273 VCMPacketizationCallback* transport) {
274 CriticalSectionScoped cs(_sendCritSect);
275 _encodedFrameCallback.SetMediaOpt(&_mediaOpt);
276 _encodedFrameCallback.SetTransportCallback(transport);
277 return VCM_OK;
278 }
279
280 // Register video output information callback which will be called to deliver
281 // information about the video stream produced by the encoder, for instance the
282 // average frame rate and bit rate.
RegisterSendStatisticsCallback(VCMSendStatisticsCallback * sendStats)283 int32_t VideoSender::RegisterSendStatisticsCallback(
284 VCMSendStatisticsCallback* sendStats) {
285 CriticalSectionScoped cs(process_crit_sect_.get());
286 _sendStatsCallback = sendStats;
287 return VCM_OK;
288 }
289
290 // Register a video quality settings callback which will be called when frame
291 // rate/dimensions need to be updated for video quality optimization
RegisterVideoQMCallback(VCMQMSettingsCallback * qm_settings_callback)292 int32_t VideoSender::RegisterVideoQMCallback(
293 VCMQMSettingsCallback* qm_settings_callback) {
294 CriticalSectionScoped cs(_sendCritSect);
295 qm_settings_callback_ = qm_settings_callback;
296 _mediaOpt.EnableQM(qm_settings_callback_ != NULL);
297 return VCM_OK;
298 }
299
300 // Register a video protection callback which will be called to deliver the
301 // requested FEC rate and NACK status (on/off).
RegisterProtectionCallback(VCMProtectionCallback * protection_callback)302 int32_t VideoSender::RegisterProtectionCallback(
303 VCMProtectionCallback* protection_callback) {
304 CriticalSectionScoped cs(_sendCritSect);
305 protection_callback_ = protection_callback;
306 return VCM_OK;
307 }
308
309 // Enable or disable a video protection method.
310 // Note: This API should be deprecated, as it does not offer a distinction
311 // between the protection method and decoding with or without errors. If such a
312 // behavior is desired, use the following API: SetReceiverRobustnessMode.
SetVideoProtection(VCMVideoProtection videoProtection,bool enable)313 int32_t VideoSender::SetVideoProtection(VCMVideoProtection videoProtection,
314 bool enable) {
315 switch (videoProtection) {
316 case kProtectionNack:
317 case kProtectionNackSender: {
318 CriticalSectionScoped cs(_sendCritSect);
319 _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNack);
320 break;
321 }
322
323 case kProtectionNackFEC: {
324 CriticalSectionScoped cs(_sendCritSect);
325 _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNackFec);
326 break;
327 }
328
329 case kProtectionFEC: {
330 CriticalSectionScoped cs(_sendCritSect);
331 _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
332 break;
333 }
334
335 case kProtectionPeriodicKeyFrames: {
336 CriticalSectionScoped cs(_sendCritSect);
337 return _codecDataBase.SetPeriodicKeyFrames(enable) ? 0 : -1;
338 break;
339 }
340 case kProtectionNackReceiver:
341 case kProtectionDualDecoder:
342 case kProtectionKeyOnLoss:
343 case kProtectionKeyOnKeyLoss:
344 // Ignore decoder modes.
345 return VCM_OK;
346 }
347 return VCM_OK;
348 }
349 // Add one raw video frame to the encoder, blocking.
AddVideoFrame(const I420VideoFrame & videoFrame,const VideoContentMetrics * contentMetrics,const CodecSpecificInfo * codecSpecificInfo)350 int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame,
351 const VideoContentMetrics* contentMetrics,
352 const CodecSpecificInfo* codecSpecificInfo) {
353 CriticalSectionScoped cs(_sendCritSect);
354 if (_encoder == NULL) {
355 return VCM_UNINITIALIZED;
356 }
357 // TODO(holmer): Add support for dropping frames per stream. Currently we
358 // only have one frame dropper for all streams.
359 if (_nextFrameTypes[0] == kFrameEmpty) {
360 return VCM_OK;
361 }
362 if (_mediaOpt.DropFrame()) {
363 return VCM_OK;
364 }
365 _mediaOpt.UpdateContentData(contentMetrics);
366 int32_t ret =
367 _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes);
368 recorder_->Add(videoFrame);
369 if (ret < 0) {
370 LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
371 return ret;
372 }
373 for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
374 _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type.
375 }
376 return VCM_OK;
377 }
378
IntraFrameRequest(int stream_index)379 int32_t VideoSender::IntraFrameRequest(int stream_index) {
380 CriticalSectionScoped cs(_sendCritSect);
381 if (stream_index < 0 ||
382 static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) {
383 return -1;
384 }
385 _nextFrameTypes[stream_index] = kVideoFrameKey;
386 if (_encoder != NULL && _encoder->InternalSource()) {
387 // Try to request the frame if we have an external encoder with
388 // internal source since AddVideoFrame never will be called.
389 if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) {
390 _nextFrameTypes[stream_index] = kVideoFrameDelta;
391 }
392 }
393 return VCM_OK;
394 }
395
EnableFrameDropper(bool enable)396 int32_t VideoSender::EnableFrameDropper(bool enable) {
397 CriticalSectionScoped cs(_sendCritSect);
398 frame_dropper_enabled_ = enable;
399 _mediaOpt.EnableFrameDropper(enable);
400 return VCM_OK;
401 }
402
SetSenderNackMode(SenderNackMode mode)403 int VideoSender::SetSenderNackMode(SenderNackMode mode) {
404 CriticalSectionScoped cs(_sendCritSect);
405
406 switch (mode) {
407 case VideoCodingModule::kNackNone:
408 _mediaOpt.EnableProtectionMethod(false, media_optimization::kNack);
409 break;
410 case VideoCodingModule::kNackAll:
411 _mediaOpt.EnableProtectionMethod(true, media_optimization::kNack);
412 break;
413 case VideoCodingModule::kNackSelective:
414 return VCM_NOT_IMPLEMENTED;
415 break;
416 }
417 return VCM_OK;
418 }
419
SetSenderReferenceSelection(bool enable)420 int VideoSender::SetSenderReferenceSelection(bool enable) {
421 return VCM_NOT_IMPLEMENTED;
422 }
423
SetSenderFEC(bool enable)424 int VideoSender::SetSenderFEC(bool enable) {
425 CriticalSectionScoped cs(_sendCritSect);
426 _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
427 return VCM_OK;
428 }
429
SetSenderKeyFramePeriod(int periodMs)430 int VideoSender::SetSenderKeyFramePeriod(int periodMs) {
431 return VCM_NOT_IMPLEMENTED;
432 }
433
StartDebugRecording(const char * file_name_utf8)434 int VideoSender::StartDebugRecording(const char* file_name_utf8) {
435 return recorder_->Start(file_name_utf8);
436 }
437
StopDebugRecording()438 void VideoSender::StopDebugRecording() {
439 recorder_->Stop();
440 }
441
SuspendBelowMinBitrate()442 void VideoSender::SuspendBelowMinBitrate() {
443 CriticalSectionScoped cs(_sendCritSect);
444 VideoCodec current_send_codec;
445 if (SendCodec(¤t_send_codec) != 0) {
446 assert(false); // Must set a send codec before SuspendBelowMinBitrate.
447 return;
448 }
449 int threshold_bps;
450 if (current_send_codec.numberOfSimulcastStreams == 0) {
451 threshold_bps = current_send_codec.minBitrate * 1000;
452 } else {
453 threshold_bps = current_send_codec.simulcastStream[0].minBitrate * 1000;
454 }
455 // Set the hysteresis window to be at 10% of the threshold, but at least
456 // 10 kbps.
457 int window_bps = std::max(threshold_bps / 10, 10000);
458 _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps);
459 }
460
VideoSuspended() const461 bool VideoSender::VideoSuspended() const {
462 CriticalSectionScoped cs(_sendCritSect);
463 return _mediaOpt.IsVideoSuspended();
464 }
465 } // namespace vcm
466 } // namespace webrtc
467