1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
12
13 #include <cmath>
14 #include <memory>
15 #include <utility>
16
17 #include "absl/strings/string_view.h"
18 #include "api/video/video_codec_type.h"
19 #include "api/video_codecs/video_encoder.h"
20 #include "modules/video_coding/include/video_error_codes.h"
21 #include "rtc_base/logging.h"
22
23 namespace webrtc {
24 namespace webrtc_pc_e2e {
25 namespace {
26
27 constexpr size_t kMaxFrameInPipelineCount = 1000;
28 constexpr double kNoMultiplier = 1.0;
29 constexpr double kEps = 1e-6;
30
GetMinMaxBitratesBps(const VideoCodec & codec,size_t spatial_idx)31 std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec,
32 size_t spatial_idx) {
33 uint32_t min_bitrate = codec.minBitrate;
34 uint32_t max_bitrate = codec.maxBitrate;
35 if (spatial_idx < codec.numberOfSimulcastStreams &&
36 codec.codecType != VideoCodecType::kVideoCodecVP9) {
37 min_bitrate =
38 std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate);
39 max_bitrate =
40 std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate);
41 }
42 if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
43 spatial_idx < codec.VP9().numberOfSpatialLayers) {
44 min_bitrate =
45 std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate);
46 max_bitrate =
47 std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate);
48 }
49 RTC_DCHECK_GT(max_bitrate, min_bitrate);
50 return {min_bitrate * 1000, max_bitrate * 1000};
51 }
52
53 } // namespace
54
QualityAnalyzingVideoEncoder(int id,absl::string_view peer_name,std::unique_ptr<VideoEncoder> delegate,double bitrate_multiplier,std::map<std::string,absl::optional<int>> stream_required_spatial_index,EncodedImageDataInjector * injector,VideoQualityAnalyzerInterface * analyzer)55 QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder(
56 int id,
57 absl::string_view peer_name,
58 std::unique_ptr<VideoEncoder> delegate,
59 double bitrate_multiplier,
60 std::map<std::string, absl::optional<int>> stream_required_spatial_index,
61 EncodedImageDataInjector* injector,
62 VideoQualityAnalyzerInterface* analyzer)
63 : id_(id),
64 peer_name_(peer_name),
65 delegate_(std::move(delegate)),
66 bitrate_multiplier_(bitrate_multiplier),
67 stream_required_spatial_index_(std::move(stream_required_spatial_index)),
68 injector_(injector),
69 analyzer_(analyzer),
70 mode_(SimulcastMode::kNormal),
71 delegate_callback_(nullptr) {}
72 QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default;
73
SetFecControllerOverride(FecControllerOverride * fec_controller_override)74 void QualityAnalyzingVideoEncoder::SetFecControllerOverride(
75 FecControllerOverride* fec_controller_override) {
76 // Ignored.
77 }
78
InitEncode(const VideoCodec * codec_settings,const Settings & settings)79 int32_t QualityAnalyzingVideoEncoder::InitEncode(
80 const VideoCodec* codec_settings,
81 const Settings& settings) {
82 MutexLock lock(&lock_);
83 codec_settings_ = *codec_settings;
84 mode_ = SimulcastMode::kNormal;
85 if (codec_settings->codecType == kVideoCodecVP9) {
86 if (codec_settings->VP9().numberOfSpatialLayers > 1) {
87 switch (codec_settings->VP9().interLayerPred) {
88 case InterLayerPredMode::kOn:
89 mode_ = SimulcastMode::kSVC;
90 break;
91 case InterLayerPredMode::kOnKeyPic:
92 mode_ = SimulcastMode::kKSVC;
93 break;
94 case InterLayerPredMode::kOff:
95 mode_ = SimulcastMode::kSimulcast;
96 break;
97 default:
98 RTC_NOTREACHED() << "Unknown codec_settings->VP9().interLayerPred";
99 break;
100 }
101 }
102 }
103 if (codec_settings->numberOfSimulcastStreams > 1) {
104 mode_ = SimulcastMode::kSimulcast;
105 }
106 return delegate_->InitEncode(codec_settings, settings);
107 }
108
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)109 int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback(
110 EncodedImageCallback* callback) {
111 // We need to get a lock here because delegate_callback can be hypothetically
112 // accessed from different thread (encoder one) concurrently.
113 MutexLock lock(&lock_);
114 delegate_callback_ = callback;
115 return delegate_->RegisterEncodeCompleteCallback(this);
116 }
117
Release()118 int32_t QualityAnalyzingVideoEncoder::Release() {
119 // Release encoder first. During release process it can still encode some
120 // frames, so we don't take a lock to prevent deadlock.
121 int32_t result = delegate_->Release();
122
123 MutexLock lock(&lock_);
124 delegate_callback_ = nullptr;
125 return result;
126 }
127
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)128 int32_t QualityAnalyzingVideoEncoder::Encode(
129 const VideoFrame& frame,
130 const std::vector<VideoFrameType>* frame_types) {
131 {
132 MutexLock lock(&lock_);
133 // Store id to be able to retrieve it in analyzing callback.
134 timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()});
135 // If this list is growing, it means that we are not receiving new encoded
136 // images from encoder. So it should be a bug in setup on in the encoder.
137 RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount);
138 }
139 analyzer_->OnFramePreEncode(peer_name_, frame);
140 int32_t result = delegate_->Encode(frame, frame_types);
141 if (result != WEBRTC_VIDEO_CODEC_OK) {
142 // If origin encoder failed, then cleanup data for this frame.
143 {
144 MutexLock lock(&lock_);
145 // The timestamp-frame_id pair can be not the last one, so we need to
146 // find it first and then remove. We will search from the end, because
147 // usually it will be the last or close to the last one.
148 auto it = timestamp_to_frame_id_list_.end();
149 while (it != timestamp_to_frame_id_list_.begin()) {
150 --it;
151 if (it->first == frame.timestamp()) {
152 timestamp_to_frame_id_list_.erase(it);
153 break;
154 }
155 }
156 }
157 analyzer_->OnEncoderError(peer_name_, frame, result);
158 }
159 return result;
160 }
161
SetRates(const VideoEncoder::RateControlParameters & parameters)162 void QualityAnalyzingVideoEncoder::SetRates(
163 const VideoEncoder::RateControlParameters& parameters) {
164 RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
165 if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
166 {
167 MutexLock lock(&lock_);
168 bitrate_allocation_ = parameters.bitrate;
169 }
170 return delegate_->SetRates(parameters);
171 }
172
173 // Simulating encoder overshooting target bitrate, by configuring actual
174 // encoder too high. Take care not to adjust past limits of config,
175 // otherwise encoders may crash on DCHECK.
176 VideoBitrateAllocation multiplied_allocation;
177 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
178 const uint32_t spatial_layer_bitrate_bps =
179 parameters.bitrate.GetSpatialLayerSum(si);
180 if (spatial_layer_bitrate_bps == 0) {
181 continue;
182 }
183
184 uint32_t min_bitrate_bps;
185 uint32_t max_bitrate_bps;
186 std::tie(min_bitrate_bps, max_bitrate_bps) =
187 GetMinMaxBitratesBps(codec_settings_, si);
188 double bitrate_multiplier = bitrate_multiplier_;
189 const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
190 bitrate_multiplier * spatial_layer_bitrate_bps);
191 if (corrected_bitrate < min_bitrate_bps) {
192 bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
193 } else if (corrected_bitrate > max_bitrate_bps) {
194 bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
195 }
196
197 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
198 if (parameters.bitrate.HasBitrate(si, ti)) {
199 multiplied_allocation.SetBitrate(
200 si, ti,
201 rtc::checked_cast<uint32_t>(bitrate_multiplier *
202 parameters.bitrate.GetBitrate(si, ti)));
203 }
204 }
205 }
206
207 RateControlParameters adjusted_params = parameters;
208 adjusted_params.bitrate = multiplied_allocation;
209 {
210 MutexLock lock(&lock_);
211 bitrate_allocation_ = adjusted_params.bitrate;
212 }
213 return delegate_->SetRates(adjusted_params);
214 }
215
GetEncoderInfo() const216 VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const {
217 return delegate_->GetEncoderInfo();
218 }
219
220 // It is assumed, that encoded callback will be always invoked with encoded
221 // images that correspond to the frames in the same sequence, that frames
222 // arrived. In other words, assume we have frames F1, F2 and F3 and they have
223 // corresponding encoded images I1, I2 and I3. In such case if we will call
224 // encode first with F1, then with F2 and then with F3, then encoder callback
225 // will be called first with all spatial layers for F1 (I1), then F2 (I2) and
226 // then F3 (I3).
227 //
228 // Basing on it we will use a list of timestamp-frame_id pairs like this:
229 // 1. If current encoded image timestamp is equals to timestamp in the front
230 // pair - pick frame id from that pair
231 // 2. If current encoded image timestamp isn't equals to timestamp in the front
232 // pair - remove the front pair and got to the step 1.
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)233 EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
234 const EncodedImage& encoded_image,
235 const CodecSpecificInfo* codec_specific_info,
236 const RTPFragmentationHeader* fragmentation) {
237 uint16_t frame_id;
238 bool discard = false;
239 uint32_t target_encode_bitrate = 0;
240 {
241 MutexLock lock(&lock_);
242 std::pair<uint32_t, uint16_t> timestamp_frame_id;
243 while (!timestamp_to_frame_id_list_.empty()) {
244 timestamp_frame_id = timestamp_to_frame_id_list_.front();
245 if (timestamp_frame_id.first == encoded_image.Timestamp()) {
246 break;
247 }
248 timestamp_to_frame_id_list_.pop_front();
249 }
250
251 // After the loop the first element should point to current |encoded_image|
252 // frame id. We don't remove it from the list, because there may be
253 // multiple spatial layers for this frame, so encoder can produce more
254 // encoded images with this timestamp. The first element will be removed
255 // when the next frame would be encoded and EncodedImageCallback would be
256 // called with the next timestamp.
257
258 if (timestamp_to_frame_id_list_.empty()) {
259 // Ensure, that we have info about this frame. It can happen that for some
260 // reasons encoder response, that he failed to decode, when we were
261 // posting frame to it, but then call the callback for this frame.
262 RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No "
263 "frame id for encoded_image.Timestamp()="
264 << encoded_image.Timestamp();
265 return EncodedImageCallback::Result(
266 EncodedImageCallback::Result::Error::OK);
267 }
268 frame_id = timestamp_frame_id.second;
269
270 discard = ShouldDiscard(frame_id, encoded_image);
271 if (!discard) {
272 target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
273 encoded_image.SpatialIndex().value_or(0));
274 }
275 }
276
277 if (!discard) {
278 // Analyzer should see only encoded images, that weren't discarded. But all
279 // not discarded layers have to be passed.
280 VideoQualityAnalyzerInterface::EncoderStats stats;
281 stats.target_encode_bitrate = target_encode_bitrate;
282 analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats);
283 }
284
285 // Image data injector injects frame id and discard flag into provided
286 // EncodedImage and returns the image with a) modified original buffer (in
287 // such case the current owner of the buffer will be responsible for deleting
288 // it) or b) a new buffer (in such case injector will be responsible for
289 // deleting it).
290 const EncodedImage& image =
291 injector_->InjectData(frame_id, discard, encoded_image, id_);
292 {
293 MutexLock lock(&lock_);
294 RTC_DCHECK(delegate_callback_);
295 return delegate_callback_->OnEncodedImage(image, codec_specific_info,
296 fragmentation);
297 }
298 }
299
OnDroppedFrame(EncodedImageCallback::DropReason reason)300 void QualityAnalyzingVideoEncoder::OnDroppedFrame(
301 EncodedImageCallback::DropReason reason) {
302 MutexLock lock(&lock_);
303 analyzer_->OnFrameDropped(peer_name_, reason);
304 RTC_DCHECK(delegate_callback_);
305 delegate_callback_->OnDroppedFrame(reason);
306 }
307
ShouldDiscard(uint16_t frame_id,const EncodedImage & encoded_image)308 bool QualityAnalyzingVideoEncoder::ShouldDiscard(
309 uint16_t frame_id,
310 const EncodedImage& encoded_image) {
311 std::string stream_label = analyzer_->GetStreamLabel(frame_id);
312 absl::optional<int> required_spatial_index =
313 stream_required_spatial_index_[stream_label];
314 if (required_spatial_index) {
315 if (*required_spatial_index == kAnalyzeAnySpatialStream) {
316 return false;
317 }
318 absl::optional<int> cur_spatial_index = encoded_image.SpatialIndex();
319 if (!cur_spatial_index) {
320 cur_spatial_index = 0;
321 }
322 RTC_CHECK(mode_ != SimulcastMode::kNormal)
323 << "Analyzing encoder is in kNormal "
324 "mode, but spatial layer/simulcast "
325 "stream met.";
326 if (mode_ == SimulcastMode::kSimulcast) {
327 // In simulcast mode only encoded images with required spatial index are
328 // interested, so all others have to be discarded.
329 return *cur_spatial_index != *required_spatial_index;
330 } else if (mode_ == SimulcastMode::kSVC) {
331 // In SVC mode encoded images with spatial indexes that are equal or
332 // less than required one are interesting, so all above have to be
333 // discarded.
334 return *cur_spatial_index > *required_spatial_index;
335 } else if (mode_ == SimulcastMode::kKSVC) {
336 // In KSVC mode for key frame encoded images with spatial indexes that
337 // are equal or less than required one are interesting, so all above
338 // have to be discarded. For other frames only required spatial index
339 // is interesting, so all others have to be discarded.
340 if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
341 return *cur_spatial_index > *required_spatial_index;
342 } else {
343 return *cur_spatial_index != *required_spatial_index;
344 }
345 } else {
346 RTC_NOTREACHED() << "Unsupported encoder mode";
347 }
348 }
349 return false;
350 }
351
QualityAnalyzingVideoEncoderFactory(absl::string_view peer_name,std::unique_ptr<VideoEncoderFactory> delegate,double bitrate_multiplier,std::map<std::string,absl::optional<int>> stream_required_spatial_index,IdGenerator<int> * id_generator,EncodedImageDataInjector * injector,VideoQualityAnalyzerInterface * analyzer)352 QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory(
353 absl::string_view peer_name,
354 std::unique_ptr<VideoEncoderFactory> delegate,
355 double bitrate_multiplier,
356 std::map<std::string, absl::optional<int>> stream_required_spatial_index,
357 IdGenerator<int>* id_generator,
358 EncodedImageDataInjector* injector,
359 VideoQualityAnalyzerInterface* analyzer)
360 : peer_name_(peer_name),
361 delegate_(std::move(delegate)),
362 bitrate_multiplier_(bitrate_multiplier),
363 stream_required_spatial_index_(std::move(stream_required_spatial_index)),
364 id_generator_(id_generator),
365 injector_(injector),
366 analyzer_(analyzer) {}
367 QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() =
368 default;
369
370 std::vector<SdpVideoFormat>
GetSupportedFormats() const371 QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const {
372 return delegate_->GetSupportedFormats();
373 }
374
375 VideoEncoderFactory::CodecInfo
QueryVideoEncoder(const SdpVideoFormat & format) const376 QualityAnalyzingVideoEncoderFactory::QueryVideoEncoder(
377 const SdpVideoFormat& format) const {
378 return delegate_->QueryVideoEncoder(format);
379 }
380
381 std::unique_ptr<VideoEncoder>
CreateVideoEncoder(const SdpVideoFormat & format)382 QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder(
383 const SdpVideoFormat& format) {
384 return std::make_unique<QualityAnalyzingVideoEncoder>(
385 id_generator_->GetNextId(), peer_name_,
386 delegate_->CreateVideoEncoder(format), bitrate_multiplier_,
387 stream_required_spatial_index_, injector_, analyzer_);
388 }
389
390 } // namespace webrtc_pc_e2e
391 } // namespace webrtc
392