• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
12 
13 #include <cmath>
14 #include <memory>
15 #include <utility>
16 
17 #include "absl/strings/string_view.h"
18 #include "api/video/video_codec_type.h"
19 #include "api/video_codecs/video_encoder.h"
20 #include "modules/video_coding/include/video_error_codes.h"
21 #include "rtc_base/logging.h"
22 
23 namespace webrtc {
24 namespace webrtc_pc_e2e {
25 namespace {
26 
27 constexpr size_t kMaxFrameInPipelineCount = 1000;
28 constexpr double kNoMultiplier = 1.0;
29 constexpr double kEps = 1e-6;
30 
GetMinMaxBitratesBps(const VideoCodec & codec,size_t spatial_idx)31 std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec,
32                                                    size_t spatial_idx) {
33   uint32_t min_bitrate = codec.minBitrate;
34   uint32_t max_bitrate = codec.maxBitrate;
35   if (spatial_idx < codec.numberOfSimulcastStreams &&
36       codec.codecType != VideoCodecType::kVideoCodecVP9) {
37     min_bitrate =
38         std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate);
39     max_bitrate =
40         std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate);
41   }
42   if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
43       spatial_idx < codec.VP9().numberOfSpatialLayers) {
44     min_bitrate =
45         std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate);
46     max_bitrate =
47         std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate);
48   }
49   RTC_DCHECK_GT(max_bitrate, min_bitrate);
50   return {min_bitrate * 1000, max_bitrate * 1000};
51 }
52 
53 }  // namespace
54 
QualityAnalyzingVideoEncoder(int id,absl::string_view peer_name,std::unique_ptr<VideoEncoder> delegate,double bitrate_multiplier,std::map<std::string,absl::optional<int>> stream_required_spatial_index,EncodedImageDataInjector * injector,VideoQualityAnalyzerInterface * analyzer)55 QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder(
56     int id,
57     absl::string_view peer_name,
58     std::unique_ptr<VideoEncoder> delegate,
59     double bitrate_multiplier,
60     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
61     EncodedImageDataInjector* injector,
62     VideoQualityAnalyzerInterface* analyzer)
63     : id_(id),
64       peer_name_(peer_name),
65       delegate_(std::move(delegate)),
66       bitrate_multiplier_(bitrate_multiplier),
67       stream_required_spatial_index_(std::move(stream_required_spatial_index)),
68       injector_(injector),
69       analyzer_(analyzer),
70       mode_(SimulcastMode::kNormal),
71       delegate_callback_(nullptr) {}
72 QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default;
73 
SetFecControllerOverride(FecControllerOverride * fec_controller_override)74 void QualityAnalyzingVideoEncoder::SetFecControllerOverride(
75     FecControllerOverride* fec_controller_override) {
76   // Ignored.
77 }
78 
InitEncode(const VideoCodec * codec_settings,const Settings & settings)79 int32_t QualityAnalyzingVideoEncoder::InitEncode(
80     const VideoCodec* codec_settings,
81     const Settings& settings) {
82   MutexLock lock(&lock_);
83   codec_settings_ = *codec_settings;
84   mode_ = SimulcastMode::kNormal;
85   if (codec_settings->codecType == kVideoCodecVP9) {
86     if (codec_settings->VP9().numberOfSpatialLayers > 1) {
87       switch (codec_settings->VP9().interLayerPred) {
88         case InterLayerPredMode::kOn:
89           mode_ = SimulcastMode::kSVC;
90           break;
91         case InterLayerPredMode::kOnKeyPic:
92           mode_ = SimulcastMode::kKSVC;
93           break;
94         case InterLayerPredMode::kOff:
95           mode_ = SimulcastMode::kSimulcast;
96           break;
97         default:
98           RTC_NOTREACHED() << "Unknown codec_settings->VP9().interLayerPred";
99           break;
100       }
101     }
102   }
103   if (codec_settings->numberOfSimulcastStreams > 1) {
104     mode_ = SimulcastMode::kSimulcast;
105   }
106   return delegate_->InitEncode(codec_settings, settings);
107 }
108 
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)109 int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback(
110     EncodedImageCallback* callback) {
111   // We need to get a lock here because delegate_callback can be hypothetically
112   // accessed from different thread (encoder one) concurrently.
113   MutexLock lock(&lock_);
114   delegate_callback_ = callback;
115   return delegate_->RegisterEncodeCompleteCallback(this);
116 }
117 
Release()118 int32_t QualityAnalyzingVideoEncoder::Release() {
119   // Release encoder first. During release process it can still encode some
120   // frames, so we don't take a lock to prevent deadlock.
121   int32_t result = delegate_->Release();
122 
123   MutexLock lock(&lock_);
124   delegate_callback_ = nullptr;
125   return result;
126 }
127 
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)128 int32_t QualityAnalyzingVideoEncoder::Encode(
129     const VideoFrame& frame,
130     const std::vector<VideoFrameType>* frame_types) {
131   {
132     MutexLock lock(&lock_);
133     // Store id to be able to retrieve it in analyzing callback.
134     timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()});
135     // If this list is growing, it means that we are not receiving new encoded
136     // images from encoder. So it should be a bug in setup on in the encoder.
137     RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount);
138   }
139   analyzer_->OnFramePreEncode(peer_name_, frame);
140   int32_t result = delegate_->Encode(frame, frame_types);
141   if (result != WEBRTC_VIDEO_CODEC_OK) {
142     // If origin encoder failed, then cleanup data for this frame.
143     {
144       MutexLock lock(&lock_);
145       // The timestamp-frame_id pair can be not the last one, so we need to
146       // find it first and then remove. We will search from the end, because
147       // usually it will be the last or close to the last one.
148       auto it = timestamp_to_frame_id_list_.end();
149       while (it != timestamp_to_frame_id_list_.begin()) {
150         --it;
151         if (it->first == frame.timestamp()) {
152           timestamp_to_frame_id_list_.erase(it);
153           break;
154         }
155       }
156     }
157     analyzer_->OnEncoderError(peer_name_, frame, result);
158   }
159   return result;
160 }
161 
SetRates(const VideoEncoder::RateControlParameters & parameters)162 void QualityAnalyzingVideoEncoder::SetRates(
163     const VideoEncoder::RateControlParameters& parameters) {
164   RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
165   if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
166     {
167       MutexLock lock(&lock_);
168       bitrate_allocation_ = parameters.bitrate;
169     }
170     return delegate_->SetRates(parameters);
171   }
172 
173   // Simulating encoder overshooting target bitrate, by configuring actual
174   // encoder too high. Take care not to adjust past limits of config,
175   // otherwise encoders may crash on DCHECK.
176   VideoBitrateAllocation multiplied_allocation;
177   for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
178     const uint32_t spatial_layer_bitrate_bps =
179         parameters.bitrate.GetSpatialLayerSum(si);
180     if (spatial_layer_bitrate_bps == 0) {
181       continue;
182     }
183 
184     uint32_t min_bitrate_bps;
185     uint32_t max_bitrate_bps;
186     std::tie(min_bitrate_bps, max_bitrate_bps) =
187         GetMinMaxBitratesBps(codec_settings_, si);
188     double bitrate_multiplier = bitrate_multiplier_;
189     const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
190         bitrate_multiplier * spatial_layer_bitrate_bps);
191     if (corrected_bitrate < min_bitrate_bps) {
192       bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
193     } else if (corrected_bitrate > max_bitrate_bps) {
194       bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
195     }
196 
197     for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
198       if (parameters.bitrate.HasBitrate(si, ti)) {
199         multiplied_allocation.SetBitrate(
200             si, ti,
201             rtc::checked_cast<uint32_t>(bitrate_multiplier *
202                                         parameters.bitrate.GetBitrate(si, ti)));
203       }
204     }
205   }
206 
207   RateControlParameters adjusted_params = parameters;
208   adjusted_params.bitrate = multiplied_allocation;
209   {
210     MutexLock lock(&lock_);
211     bitrate_allocation_ = adjusted_params.bitrate;
212   }
213   return delegate_->SetRates(adjusted_params);
214 }
215 
GetEncoderInfo() const216 VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const {
217   return delegate_->GetEncoderInfo();
218 }
219 
220 // It is assumed, that encoded callback will be always invoked with encoded
221 // images that correspond to the frames in the same sequence, that frames
222 // arrived. In other words, assume we have frames F1, F2 and F3 and they have
223 // corresponding encoded images I1, I2 and I3. In such case if we will call
224 // encode first with F1, then with F2 and then with F3, then encoder callback
225 // will be called first with all spatial layers for F1 (I1), then F2 (I2) and
226 // then F3 (I3).
227 //
228 // Basing on it we will use a list of timestamp-frame_id pairs like this:
229 //  1. If current encoded image timestamp is equals to timestamp in the front
230 //     pair - pick frame id from that pair
231 //  2. If current encoded image timestamp isn't equals to timestamp in the front
232 //     pair - remove the front pair and got to the step 1.
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)233 EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
234     const EncodedImage& encoded_image,
235     const CodecSpecificInfo* codec_specific_info,
236     const RTPFragmentationHeader* fragmentation) {
237   uint16_t frame_id;
238   bool discard = false;
239   uint32_t target_encode_bitrate = 0;
240   {
241     MutexLock lock(&lock_);
242     std::pair<uint32_t, uint16_t> timestamp_frame_id;
243     while (!timestamp_to_frame_id_list_.empty()) {
244       timestamp_frame_id = timestamp_to_frame_id_list_.front();
245       if (timestamp_frame_id.first == encoded_image.Timestamp()) {
246         break;
247       }
248       timestamp_to_frame_id_list_.pop_front();
249     }
250 
251     // After the loop the first element should point to current |encoded_image|
252     // frame id. We don't remove it from the list, because there may be
253     // multiple spatial layers for this frame, so encoder can produce more
254     // encoded images with this timestamp. The first element will be removed
255     // when the next frame would be encoded and EncodedImageCallback would be
256     // called with the next timestamp.
257 
258     if (timestamp_to_frame_id_list_.empty()) {
259       // Ensure, that we have info about this frame. It can happen that for some
260       // reasons encoder response, that he failed to decode, when we were
261       // posting frame to it, but then call the callback for this frame.
262       RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No "
263                            "frame id for encoded_image.Timestamp()="
264                         << encoded_image.Timestamp();
265       return EncodedImageCallback::Result(
266           EncodedImageCallback::Result::Error::OK);
267     }
268     frame_id = timestamp_frame_id.second;
269 
270     discard = ShouldDiscard(frame_id, encoded_image);
271     if (!discard) {
272       target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
273           encoded_image.SpatialIndex().value_or(0));
274     }
275   }
276 
277   if (!discard) {
278     // Analyzer should see only encoded images, that weren't discarded. But all
279     // not discarded layers have to be passed.
280     VideoQualityAnalyzerInterface::EncoderStats stats;
281     stats.target_encode_bitrate = target_encode_bitrate;
282     analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats);
283   }
284 
285   // Image data injector injects frame id and discard flag into provided
286   // EncodedImage and returns the image with a) modified original buffer (in
287   // such case the current owner of the buffer will be responsible for deleting
288   // it) or b) a new buffer (in such case injector will be responsible for
289   // deleting it).
290   const EncodedImage& image =
291       injector_->InjectData(frame_id, discard, encoded_image, id_);
292   {
293     MutexLock lock(&lock_);
294     RTC_DCHECK(delegate_callback_);
295     return delegate_callback_->OnEncodedImage(image, codec_specific_info,
296                                               fragmentation);
297   }
298 }
299 
OnDroppedFrame(EncodedImageCallback::DropReason reason)300 void QualityAnalyzingVideoEncoder::OnDroppedFrame(
301     EncodedImageCallback::DropReason reason) {
302   MutexLock lock(&lock_);
303   analyzer_->OnFrameDropped(peer_name_, reason);
304   RTC_DCHECK(delegate_callback_);
305   delegate_callback_->OnDroppedFrame(reason);
306 }
307 
ShouldDiscard(uint16_t frame_id,const EncodedImage & encoded_image)308 bool QualityAnalyzingVideoEncoder::ShouldDiscard(
309     uint16_t frame_id,
310     const EncodedImage& encoded_image) {
311   std::string stream_label = analyzer_->GetStreamLabel(frame_id);
312   absl::optional<int> required_spatial_index =
313       stream_required_spatial_index_[stream_label];
314   if (required_spatial_index) {
315     if (*required_spatial_index == kAnalyzeAnySpatialStream) {
316       return false;
317     }
318     absl::optional<int> cur_spatial_index = encoded_image.SpatialIndex();
319     if (!cur_spatial_index) {
320       cur_spatial_index = 0;
321     }
322     RTC_CHECK(mode_ != SimulcastMode::kNormal)
323         << "Analyzing encoder is in kNormal "
324            "mode, but spatial layer/simulcast "
325            "stream met.";
326     if (mode_ == SimulcastMode::kSimulcast) {
327       // In simulcast mode only encoded images with required spatial index are
328       // interested, so all others have to be discarded.
329       return *cur_spatial_index != *required_spatial_index;
330     } else if (mode_ == SimulcastMode::kSVC) {
331       // In SVC mode encoded images with spatial indexes that are equal or
332       // less than required one are interesting, so all above have to be
333       // discarded.
334       return *cur_spatial_index > *required_spatial_index;
335     } else if (mode_ == SimulcastMode::kKSVC) {
336       // In KSVC mode for key frame encoded images with spatial indexes that
337       // are equal or less than required one are interesting, so all above
338       // have to be discarded. For other frames only required spatial index
339       // is interesting, so all others have to be discarded.
340       if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
341         return *cur_spatial_index > *required_spatial_index;
342       } else {
343         return *cur_spatial_index != *required_spatial_index;
344       }
345     } else {
346       RTC_NOTREACHED() << "Unsupported encoder mode";
347     }
348   }
349   return false;
350 }
351 
QualityAnalyzingVideoEncoderFactory(absl::string_view peer_name,std::unique_ptr<VideoEncoderFactory> delegate,double bitrate_multiplier,std::map<std::string,absl::optional<int>> stream_required_spatial_index,IdGenerator<int> * id_generator,EncodedImageDataInjector * injector,VideoQualityAnalyzerInterface * analyzer)352 QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory(
353     absl::string_view peer_name,
354     std::unique_ptr<VideoEncoderFactory> delegate,
355     double bitrate_multiplier,
356     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
357     IdGenerator<int>* id_generator,
358     EncodedImageDataInjector* injector,
359     VideoQualityAnalyzerInterface* analyzer)
360     : peer_name_(peer_name),
361       delegate_(std::move(delegate)),
362       bitrate_multiplier_(bitrate_multiplier),
363       stream_required_spatial_index_(std::move(stream_required_spatial_index)),
364       id_generator_(id_generator),
365       injector_(injector),
366       analyzer_(analyzer) {}
367 QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() =
368     default;
369 
370 std::vector<SdpVideoFormat>
GetSupportedFormats() const371 QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const {
372   return delegate_->GetSupportedFormats();
373 }
374 
375 VideoEncoderFactory::CodecInfo
QueryVideoEncoder(const SdpVideoFormat & format) const376 QualityAnalyzingVideoEncoderFactory::QueryVideoEncoder(
377     const SdpVideoFormat& format) const {
378   return delegate_->QueryVideoEncoder(format);
379 }
380 
381 std::unique_ptr<VideoEncoder>
CreateVideoEncoder(const SdpVideoFormat & format)382 QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder(
383     const SdpVideoFormat& format) {
384   return std::make_unique<QualityAnalyzingVideoEncoder>(
385       id_generator_->GetNextId(), peer_name_,
386       delegate_->CreateVideoEncoder(format), bitrate_multiplier_,
387       stream_required_spatial_index_, injector_, analyzer_);
388 }
389 
390 }  // namespace webrtc_pc_e2e
391 }  // namespace webrtc
392