1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 13 #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 14 15 #include <atomic> 16 #include <list> 17 #include <memory> 18 #include <stack> 19 #include <string> 20 #include <utility> 21 #include <vector> 22 23 #include "absl/types/optional.h" 24 #include "api/fec_controller_override.h" 25 #include "api/sequence_checker.h" 26 #include "api/video_codecs/sdp_video_format.h" 27 #include "api/video_codecs/video_encoder.h" 28 #include "api/video_codecs/video_encoder_factory.h" 29 #include "common_video/framerate_controller.h" 30 #include "modules/video_coding/include/video_codec_interface.h" 31 #include "rtc_base/experiments/encoder_info_settings.h" 32 #include "rtc_base/system/no_unique_address.h" 33 #include "rtc_base/system/rtc_export.h" 34 35 namespace webrtc { 36 37 // SimulcastEncoderAdapter implements simulcast support by creating multiple 38 // webrtc::VideoEncoder instances with the given VideoEncoderFactory. 39 // The object is created and destroyed on the worker thread, but all public 40 // interfaces should be called from the encoder task queue. 41 class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { 42 public: 43 // TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone. 44 SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory, 45 const SdpVideoFormat& format); 46 // `primary_factory` produces the first-choice encoders to use. 47 // `fallback_factory`, if non-null, is used to create fallback encoder that 48 // will be used if InitEncode() fails for the primary encoder. 49 SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory, 50 VideoEncoderFactory* fallback_factory, 51 const SdpVideoFormat& format); 52 ~SimulcastEncoderAdapter() override; 53 54 // Implements VideoEncoder. 55 void SetFecControllerOverride( 56 FecControllerOverride* fec_controller_override) override; 57 int Release() override; 58 int InitEncode(const VideoCodec* codec_settings, 59 const VideoEncoder::Settings& settings) override; 60 int Encode(const VideoFrame& input_image, 61 const std::vector<VideoFrameType>* frame_types) override; 62 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; 63 void SetRates(const RateControlParameters& parameters) override; 64 void OnPacketLossRateUpdate(float packet_loss_rate) override; 65 void OnRttUpdate(int64_t rtt_ms) override; 66 void OnLossNotification(const LossNotification& loss_notification) override; 67 68 EncoderInfo GetEncoderInfo() const override; 69 70 private: 71 class EncoderContext { 72 public: 73 EncoderContext(std::unique_ptr<VideoEncoder> encoder, 74 bool prefer_temporal_support, 75 VideoEncoder::EncoderInfo primary_info, 76 VideoEncoder::EncoderInfo fallback_info); 77 EncoderContext& operator=(EncoderContext&&) = delete; 78 encoder()79 VideoEncoder& encoder() { return *encoder_; } prefer_temporal_support()80 bool prefer_temporal_support() { return prefer_temporal_support_; } 81 void Release(); 82 PrimaryInfo()83 const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; } 84 FallbackInfo()85 const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; } 86 87 private: 88 std::unique_ptr<VideoEncoder> encoder_; 89 bool prefer_temporal_support_; 90 const VideoEncoder::EncoderInfo primary_info_; 91 const VideoEncoder::EncoderInfo fallback_info_; 92 }; 93 94 class StreamContext : public EncodedImageCallback { 95 public: 96 StreamContext(SimulcastEncoderAdapter* parent, 97 std::unique_ptr<EncoderContext> encoder_context, 98 std::unique_ptr<FramerateController> framerate_controller, 99 int stream_idx, 100 uint16_t width, 101 uint16_t height, 102 bool send_stream); 103 StreamContext(StreamContext&& rhs); 104 StreamContext& operator=(StreamContext&&) = delete; 105 ~StreamContext() override; 106 107 Result OnEncodedImage( 108 const EncodedImage& encoded_image, 109 const CodecSpecificInfo* codec_specific_info) override; 110 void OnDroppedFrame(DropReason reason) override; 111 encoder()112 VideoEncoder& encoder() { return encoder_context_->encoder(); } encoder()113 const VideoEncoder& encoder() const { return encoder_context_->encoder(); } stream_idx()114 int stream_idx() const { return stream_idx_; } width()115 uint16_t width() const { return width_; } height()116 uint16_t height() const { return height_; } is_keyframe_needed()117 bool is_keyframe_needed() const { 118 return !is_paused_ && is_keyframe_needed_; 119 } set_is_keyframe_needed()120 void set_is_keyframe_needed() { is_keyframe_needed_ = true; } is_paused()121 bool is_paused() const { return is_paused_; } set_is_paused(bool is_paused)122 void set_is_paused(bool is_paused) { is_paused_ = is_paused; } target_fps()123 absl::optional<double> target_fps() const { 124 return framerate_controller_ == nullptr 125 ? absl::nullopt 126 : absl::optional<double>( 127 framerate_controller_->GetMaxFramerate()); 128 } 129 130 std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&; 131 void OnKeyframe(Timestamp timestamp); 132 bool ShouldDropFrame(Timestamp timestamp); 133 134 private: 135 SimulcastEncoderAdapter* const parent_; 136 std::unique_ptr<EncoderContext> encoder_context_; 137 std::unique_ptr<FramerateController> framerate_controller_; 138 const int stream_idx_; 139 const uint16_t width_; 140 const uint16_t height_; 141 bool is_keyframe_needed_; 142 bool is_paused_; 143 }; 144 145 bool Initialized() const; 146 147 void DestroyStoredEncoders(); 148 149 // This method creates encoder. May reuse previously created encoders from 150 // `cached_encoder_contexts_`. It's const because it's used from 151 // const GetEncoderInfo(). 152 std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext( 153 bool is_lowest_quality_stream) const; 154 155 webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec, 156 int stream_idx, 157 uint32_t start_bitrate_kbps, 158 bool is_lowest_quality_stream, 159 bool is_highest_quality_stream); 160 161 EncodedImageCallback::Result OnEncodedImage( 162 size_t stream_idx, 163 const EncodedImage& encoded_image, 164 const CodecSpecificInfo* codec_specific_info); 165 166 void OnDroppedFrame(size_t stream_idx); 167 168 void OverrideFromFieldTrial(VideoEncoder::EncoderInfo* info) const; 169 170 std::atomic<int> inited_; 171 VideoEncoderFactory* const primary_encoder_factory_; 172 VideoEncoderFactory* const fallback_encoder_factory_; 173 const SdpVideoFormat video_format_; 174 VideoCodec codec_; 175 int total_streams_count_; 176 bool bypass_mode_; 177 std::vector<StreamContext> stream_contexts_; 178 EncodedImageCallback* encoded_complete_callback_; 179 180 // Used for checking the single-threaded access of the encoder interface. 181 RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; 182 183 // Store previously created and released encoders , so they don't have to be 184 // recreated. Remaining encoders are destroyed by the destructor. 185 // Marked as `mutable` becuase we may need to temporarily create encoder in 186 // GetEncoderInfo(), which is const. 187 mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_; 188 189 const absl::optional<unsigned int> experimental_boosted_screenshare_qp_; 190 const bool boost_base_layer_quality_; 191 const bool prefer_temporal_support_on_base_layer_; 192 193 const SimulcastEncoderAdapterEncoderInfoSettings encoder_info_override_; 194 }; 195 196 } // namespace webrtc 197 198 #endif // MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ 199