• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_VIDEO_STREAM_ENCODER_H_
12 #define VIDEO_VIDEO_STREAM_ENCODER_H_
13 
14 #include <atomic>
15 #include <map>
16 #include <memory>
17 #include <string>
18 #include <vector>
19 
20 #include "api/adaptation/resource.h"
21 #include "api/units/data_rate.h"
22 #include "api/video/video_bitrate_allocator.h"
23 #include "api/video/video_rotation.h"
24 #include "api/video/video_sink_interface.h"
25 #include "api/video/video_stream_encoder_interface.h"
26 #include "api/video/video_stream_encoder_observer.h"
27 #include "api/video/video_stream_encoder_settings.h"
28 #include "api/video_codecs/video_codec.h"
29 #include "api/video_codecs/video_encoder.h"
30 #include "call/adaptation/adaptation_constraint.h"
31 #include "call/adaptation/adaptation_listener.h"
32 #include "call/adaptation/resource_adaptation_processor.h"
33 #include "call/adaptation/resource_adaptation_processor_interface.h"
34 #include "call/adaptation/video_source_restrictions.h"
35 #include "call/adaptation/video_stream_input_state_provider.h"
36 #include "modules/video_coding/utility/frame_dropper.h"
37 #include "rtc_base/event.h"
38 #include "rtc_base/experiments/rate_control_settings.h"
39 #include "rtc_base/numerics/exp_filter.h"
40 #include "rtc_base/race_checker.h"
41 #include "rtc_base/rate_statistics.h"
42 #include "rtc_base/synchronization/sequence_checker.h"
43 #include "rtc_base/task_queue.h"
44 #include "rtc_base/thread_checker.h"
45 #include "system_wrappers/include/clock.h"
46 #include "video/adaptation/video_stream_encoder_resource_manager.h"
47 #include "video/encoder_bitrate_adjuster.h"
48 #include "video/frame_encode_metadata_writer.h"
49 #include "video/video_source_sink_controller.h"
50 
51 namespace webrtc {
52 
53 // VideoStreamEncoder represent a video encoder that accepts raw video frames as
54 // input and produces an encoded bit stream.
55 // Usage:
56 //  Instantiate.
57 //  Call SetSink.
58 //  Call SetSource.
59 //  Call ConfigureEncoder with the codec settings.
60 //  Call Stop() when done.
61 class VideoStreamEncoder : public VideoStreamEncoderInterface,
62                            private EncodedImageCallback,
63                            public VideoSourceRestrictionsListener {
64  public:
65   VideoStreamEncoder(Clock* clock,
66                      uint32_t number_of_cores,
67                      VideoStreamEncoderObserver* encoder_stats_observer,
68                      const VideoStreamEncoderSettings& settings,
69                      std::unique_ptr<OveruseFrameDetector> overuse_detector,
70                      TaskQueueFactory* task_queue_factory);
71   ~VideoStreamEncoder() override;
72 
73   void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
74   std::vector<rtc::scoped_refptr<Resource>> GetAdaptationResources() override;
75 
76   void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
77                  const DegradationPreference& degradation_preference) override;
78 
79   void SetSink(EncoderSink* sink, bool rotation_applied) override;
80 
81   // TODO(perkj): Can we remove VideoCodec.startBitrate ?
82   void SetStartBitrate(int start_bitrate_bps) override;
83 
84   void SetBitrateAllocationObserver(
85       VideoBitrateAllocationObserver* bitrate_observer) override;
86 
87   void SetFecControllerOverride(
88       FecControllerOverride* fec_controller_override) override;
89 
90   void ConfigureEncoder(VideoEncoderConfig config,
91                         size_t max_data_payload_length) override;
92 
93   // Permanently stop encoding. After this method has returned, it is
94   // guaranteed that no encoded frames will be delivered to the sink.
95   void Stop() override;
96 
97   void SendKeyFrame() override;
98 
99   void OnLossNotification(
100       const VideoEncoder::LossNotification& loss_notification) override;
101 
102   void OnBitrateUpdated(DataRate target_bitrate,
103                         DataRate stable_target_bitrate,
104                         DataRate target_headroom,
105                         uint8_t fraction_lost,
106                         int64_t round_trip_time_ms,
107                         double cwnd_reduce_ratio) override;
108 
109   DataRate UpdateTargetBitrate(DataRate target_bitrate,
110                                double cwnd_reduce_ratio);
111 
112  protected:
113   // Used for testing. For example the |ScalingObserverInterface| methods must
114   // be called on |encoder_queue_|.
encoder_queue()115   rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
resource_adaptation_queue()116   rtc::TaskQueue* resource_adaptation_queue() {
117     return &resource_adaptation_queue_;
118   }
119 
120   void OnVideoSourceRestrictionsUpdated(
121       VideoSourceRestrictions restrictions,
122       const VideoAdaptationCounters& adaptation_counters,
123       rtc::scoped_refptr<Resource> reason,
124       const VideoSourceRestrictions& unfiltered_restrictions) override;
125 
126   // Used for injected test resources.
127   // TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests.
128   void InjectAdaptationResource(rtc::scoped_refptr<Resource> resource,
129                                 VideoAdaptationReason reason);
130   void InjectAdaptationConstraint(AdaptationConstraint* adaptation_constraint);
131 
132   rtc::scoped_refptr<QualityScalerResource>
133   quality_scaler_resource_for_testing();
134 
135   void AddRestrictionsListenerForTesting(
136       VideoSourceRestrictionsListener* restrictions_listener);
137   void RemoveRestrictionsListenerForTesting(
138       VideoSourceRestrictionsListener* restrictions_listener);
139 
140  private:
141   class VideoFrameInfo {
142    public:
VideoFrameInfo(int width,int height,bool is_texture)143     VideoFrameInfo(int width, int height, bool is_texture)
144         : width(width), height(height), is_texture(is_texture) {}
145     int width;
146     int height;
147     bool is_texture;
pixel_count()148     int pixel_count() const { return width * height; }
149   };
150 
151   struct EncoderRateSettings {
152     EncoderRateSettings();
153     EncoderRateSettings(const VideoBitrateAllocation& bitrate,
154                         double framerate_fps,
155                         DataRate bandwidth_allocation,
156                         DataRate encoder_target,
157                         DataRate stable_encoder_target);
158     bool operator==(const EncoderRateSettings& rhs) const;
159     bool operator!=(const EncoderRateSettings& rhs) const;
160 
161     VideoEncoder::RateControlParameters rate_control;
162     // This is the scalar target bitrate before the VideoBitrateAllocator, i.e.
163     // the |target_bitrate| argument of the OnBitrateUpdated() method. This is
164     // needed because the bitrate allocator may truncate the total bitrate and a
165     // later call to the same allocator instance, e.g.
166     // |using last_encoder_rate_setings_->bitrate.get_sum_bps()|, may trick it
167     // into thinking the available bitrate has decreased since the last call.
168     DataRate encoder_target;
169     DataRate stable_encoder_target;
170   };
171 
172   class DegradationPreferenceManager;
173 
174   void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_);
175   void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_);
176 
177   // Implements VideoSinkInterface.
178   void OnFrame(const VideoFrame& video_frame) override;
179   void OnDiscardedFrame() override;
180 
181   void MaybeEncodeVideoFrame(const VideoFrame& frame,
182                              int64_t time_when_posted_in_ms);
183 
184   void EncodeVideoFrame(const VideoFrame& frame,
185                         int64_t time_when_posted_in_ms);
186   // Indicates wether frame should be dropped because the pixel count is too
187   // large for the current bitrate configuration.
188   bool DropDueToSize(uint32_t pixel_count) const RTC_RUN_ON(&encoder_queue_);
189 
190   // Implements EncodedImageCallback.
191   EncodedImageCallback::Result OnEncodedImage(
192       const EncodedImage& encoded_image,
193       const CodecSpecificInfo* codec_specific_info,
194       const RTPFragmentationHeader* fragmentation) override;
195 
196   void OnDroppedFrame(EncodedImageCallback::DropReason reason) override;
197 
198   bool EncoderPaused() const;
199   void TraceFrameDropStart();
200   void TraceFrameDropEnd();
201 
202   // Returns a copy of |rate_settings| with the |bitrate| field updated using
203   // the current VideoBitrateAllocator, and notifies any listeners of the new
204   // allocation.
205   EncoderRateSettings UpdateBitrateAllocationAndNotifyObserver(
206       const EncoderRateSettings& rate_settings) RTC_RUN_ON(&encoder_queue_);
207 
208   uint32_t GetInputFramerateFps() RTC_RUN_ON(&encoder_queue_);
209   void SetEncoderRates(const EncoderRateSettings& rate_settings)
210       RTC_RUN_ON(&encoder_queue_);
211 
212   void RunPostEncode(const EncodedImage& encoded_image,
213                      int64_t time_sent_us,
214                      int temporal_index,
215                      DataSize frame_size);
216   bool HasInternalSource() const RTC_RUN_ON(&encoder_queue_);
217   void ReleaseEncoder() RTC_RUN_ON(&encoder_queue_);
218 
219   void CheckForAnimatedContent(const VideoFrame& frame,
220                                int64_t time_when_posted_in_ms)
221       RTC_RUN_ON(&encoder_queue_);
222 
223   rtc::Event shutdown_event_;
224 
225   const uint32_t number_of_cores_;
226 
227   const bool quality_scaling_experiment_enabled_;
228 
229   EncoderSink* sink_;
230   const VideoStreamEncoderSettings settings_;
231   const RateControlSettings rate_control_settings_;
232 
233   std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface> const
234       encoder_selector_;
235   VideoStreamEncoderObserver* const encoder_stats_observer_;
236   // |thread_checker_| checks that public methods that are related to lifetime
237   // of VideoStreamEncoder are called on the same thread.
238   rtc::ThreadChecker thread_checker_;
239 
240   VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_);
241   std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_)
242       RTC_PT_GUARDED_BY(&encoder_queue_);
243   bool encoder_initialized_;
244   std::unique_ptr<VideoBitrateAllocator> rate_allocator_
245       RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_);
246   int max_framerate_ RTC_GUARDED_BY(&encoder_queue_);
247 
248   // Set when ConfigureEncoder has been called in order to lazy reconfigure the
249   // encoder on the next frame.
250   bool pending_encoder_reconfiguration_ RTC_GUARDED_BY(&encoder_queue_);
251   // Set when configuration must create a new encoder object, e.g.,
252   // because of a codec change.
253   bool pending_encoder_creation_ RTC_GUARDED_BY(&encoder_queue_);
254 
255   absl::optional<VideoFrameInfo> last_frame_info_
256       RTC_GUARDED_BY(&encoder_queue_);
257   int crop_width_ RTC_GUARDED_BY(&encoder_queue_);
258   int crop_height_ RTC_GUARDED_BY(&encoder_queue_);
259   absl::optional<uint32_t> encoder_target_bitrate_bps_
260       RTC_GUARDED_BY(&encoder_queue_);
261   size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_);
262   absl::optional<EncoderRateSettings> last_encoder_rate_settings_
263       RTC_GUARDED_BY(&encoder_queue_);
264   bool encoder_paused_and_dropped_frame_ RTC_GUARDED_BY(&encoder_queue_);
265 
266   // Set to true if at least one frame was sent to encoder since last encoder
267   // initialization.
268   bool was_encode_called_since_last_initialization_
269       RTC_GUARDED_BY(&encoder_queue_);
270 
271   bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_);
272   Clock* const clock_;
273 
274   rtc::RaceChecker incoming_frame_race_checker_
275       RTC_GUARDED_BY(incoming_frame_race_checker_);
276   std::atomic<int> posted_frames_waiting_for_encode_;
277   // Used to make sure incoming time stamp is increasing for every frame.
278   int64_t last_captured_timestamp_ RTC_GUARDED_BY(incoming_frame_race_checker_);
279   // Delta used for translating between NTP and internal timestamps.
280   const int64_t delta_ntp_internal_ms_
281       RTC_GUARDED_BY(incoming_frame_race_checker_);
282 
283   int64_t last_frame_log_ms_ RTC_GUARDED_BY(incoming_frame_race_checker_);
284   int captured_frame_count_ RTC_GUARDED_BY(&encoder_queue_);
285   int dropped_frame_cwnd_pushback_count_ RTC_GUARDED_BY(&encoder_queue_);
286   int dropped_frame_encoder_block_count_ RTC_GUARDED_BY(&encoder_queue_);
287   absl::optional<VideoFrame> pending_frame_ RTC_GUARDED_BY(&encoder_queue_);
288   int64_t pending_frame_post_time_us_ RTC_GUARDED_BY(&encoder_queue_);
289 
290   VideoFrame::UpdateRect accumulated_update_rect_
291       RTC_GUARDED_BY(&encoder_queue_);
292   bool accumulated_update_rect_is_valid_ RTC_GUARDED_BY(&encoder_queue_);
293 
294   // Used for automatic content type detection.
295   absl::optional<VideoFrame::UpdateRect> last_update_rect_
296       RTC_GUARDED_BY(&encoder_queue_);
297   Timestamp animation_start_time_ RTC_GUARDED_BY(&encoder_queue_);
298   bool cap_resolution_due_to_video_content_ RTC_GUARDED_BY(&encoder_queue_);
299   // Used to correctly ignore changes in update_rect introduced by
300   // resize triggered by animation detection.
301   enum class ExpectResizeState {
302     kNoResize,              // Normal operation.
303     kResize,                // Resize was triggered by the animation detection.
304     kFirstFrameAfterResize  // Resize observed.
305   } expect_resize_state_ RTC_GUARDED_BY(&encoder_queue_);
306 
307   VideoBitrateAllocationObserver* bitrate_observer_
308       RTC_GUARDED_BY(&encoder_queue_);
309   FecControllerOverride* fec_controller_override_
310       RTC_GUARDED_BY(&encoder_queue_);
311   absl::optional<int64_t> last_parameters_update_ms_
312       RTC_GUARDED_BY(&encoder_queue_);
313   absl::optional<int64_t> last_encode_info_ms_ RTC_GUARDED_BY(&encoder_queue_);
314 
315   VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_);
316   absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits_
317       RTC_GUARDED_BY(&encoder_queue_);
318   VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_);
319   VideoCodec send_codec_ RTC_GUARDED_BY(&encoder_queue_);
320 
321   FrameDropper frame_dropper_ RTC_GUARDED_BY(&encoder_queue_);
322   // If frame dropper is not force disabled, frame dropping might still be
323   // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a
324   // trusted rate controller. This is determined on a per-frame basis, as the
325   // encoder behavior might dynamically change.
326   bool force_disable_frame_dropper_ RTC_GUARDED_BY(&encoder_queue_);
327   RateStatistics input_framerate_ RTC_GUARDED_BY(&encoder_queue_);
328   // Incremented on worker thread whenever |frame_dropper_| determines that a
329   // frame should be dropped. Decremented on whichever thread runs
330   // OnEncodedImage(), which is only called by one thread but not necessarily
331   // the worker thread.
332   std::atomic<int> pending_frame_drops_;
333 
334   // Congestion window frame drop ratio (drop 1 in every
335   // cwnd_frame_drop_interval_ frames).
336   absl::optional<int> cwnd_frame_drop_interval_ RTC_GUARDED_BY(&encoder_queue_);
337   // Frame counter for congestion window frame drop.
338   int cwnd_frame_counter_ RTC_GUARDED_BY(&encoder_queue_);
339 
340   std::unique_ptr<EncoderBitrateAdjuster> bitrate_adjuster_
341       RTC_GUARDED_BY(&encoder_queue_);
342 
343   // TODO(sprang): Change actually support keyframe per simulcast stream, or
344   // turn this into a simple bool |pending_keyframe_request_|.
345   std::vector<VideoFrameType> next_frame_types_ RTC_GUARDED_BY(&encoder_queue_);
346 
347   FrameEncodeMetadataWriter frame_encode_metadata_writer_;
348 
349   // Experiment groups parsed from field trials for realtime video ([0]) and
350   // screenshare ([1]). 0 means no group specified. Positive values are
351   // experiment group numbers incremented by 1.
352   const std::array<uint8_t, 2> experiment_groups_;
353 
354   struct EncoderSwitchExperiment {
355     struct Thresholds {
356       absl::optional<DataRate> bitrate;
357       absl::optional<int> pixel_count;
358     };
359 
360     // Codec --> switching thresholds
361     std::map<VideoCodecType, Thresholds> codec_thresholds;
362 
363     // To smooth out the target bitrate so that we don't trigger a switch
364     // too easily.
365     rtc::ExpFilter bitrate_filter{1.0};
366 
367     // Codec/implementation to switch to
368     std::string to_codec;
369     absl::optional<std::string> to_param;
370     absl::optional<std::string> to_value;
371 
372     // Thresholds for the currently used codecs.
373     Thresholds current_thresholds;
374 
375     // Updates the |bitrate_filter|, so not const.
376     bool IsBitrateBelowThreshold(const DataRate& target_bitrate);
377     bool IsPixelCountBelowThreshold(int pixel_count) const;
378     void SetCodec(VideoCodecType codec);
379   };
380 
381   EncoderSwitchExperiment ParseEncoderSwitchFieldTrial() const;
382 
383   EncoderSwitchExperiment encoder_switch_experiment_
384       RTC_GUARDED_BY(&encoder_queue_);
385 
386   struct AutomaticAnimationDetectionExperiment {
387     bool enabled = false;
388     int min_duration_ms = 2000;
389     double min_area_ratio = 0.8;
390     int min_fps = 10;
ParserAutomaticAnimationDetectionExperiment391     std::unique_ptr<StructParametersParser> Parser() {
392       return StructParametersParser::Create(
393           "enabled", &enabled,                  //
394           "min_duration_ms", &min_duration_ms,  //
395           "min_area_ratio", &min_area_ratio,    //
396           "min_fps", &min_fps);
397     }
398   };
399 
400   AutomaticAnimationDetectionExperiment
401   ParseAutomatincAnimationDetectionFieldTrial() const;
402 
403   AutomaticAnimationDetectionExperiment
404       automatic_animation_detection_experiment_ RTC_GUARDED_BY(&encoder_queue_);
405 
406   // An encoder switch is only requested once, this variable is used to keep
407   // track of whether a request has been made or not.
408   bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
409 
410   // Provies video stream input states: current resolution and frame rate.
411   // This class is thread-safe.
412   VideoStreamInputStateProvider input_state_provider_;
413 
414   std::unique_ptr<VideoStreamAdapter> video_stream_adapter_
415       RTC_GUARDED_BY(&resource_adaptation_queue_);
416   // Responsible for adapting input resolution or frame rate to ensure resources
417   // (e.g. CPU or bandwidth) are not overused.
418   // Adding resources can occur on any thread, but all other methods need to be
419   // called on the adaptation thread.
420   std::unique_ptr<ResourceAdaptationProcessorInterface>
421       resource_adaptation_processor_;
422   std::unique_ptr<DegradationPreferenceManager> degradation_preference_manager_;
423   std::vector<AdaptationConstraint*> adaptation_constraints_
424       RTC_GUARDED_BY(&resource_adaptation_queue_);
425   // Handles input, output and stats reporting related to VideoStreamEncoder
426   // specific resources, such as "encode usage percent" measurements and "QP
427   // scaling". Also involved with various mitigations such as inital frame
428   // dropping.
429   // The manager primarily operates on the |encoder_queue_| but its lifetime is
430   // tied to the VideoStreamEncoder (which is destroyed off the encoder queue)
431   // and its resource list is accessible from any thread.
432   VideoStreamEncoderResourceManager stream_resource_manager_;
433   // Carries out the VideoSourceRestrictions provided by the
434   // ResourceAdaptationProcessor, i.e. reconfigures the source of video frames
435   // to provide us with different resolution or frame rate.
436   // This class is thread-safe.
437   VideoSourceSinkController video_source_sink_controller_;
438 
439   // Public methods are proxied to the task queues. The queues must be destroyed
440   // first to make sure no tasks run that use other members.
441   // TODO(https://crbug.com/webrtc/11172): Move ownership of the
442   // ResourceAdaptationProcessor and its task queue to Call when processors are
443   // multi-stream aware.
444   rtc::TaskQueue resource_adaptation_queue_;
445   rtc::TaskQueue encoder_queue_;
446 
447   RTC_DISALLOW_COPY_AND_ASSIGN(VideoStreamEncoder);
448 };
449 
450 }  // namespace webrtc
451 
452 #endif  // VIDEO_VIDEO_STREAM_ENCODER_H_
453