1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 * 10 */ 11 12 #ifndef MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_ 13 #define MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_ 14 15 #ifdef RTC_ENABLE_VP9 16 17 #include <array> 18 #include <memory> 19 #include <vector> 20 21 #include "api/fec_controller_override.h" 22 #include "api/field_trials_view.h" 23 #include "api/video_codecs/video_encoder.h" 24 #include "api/video_codecs/vp9_profile.h" 25 #include "common_video/include/video_frame_buffer_pool.h" 26 #include "modules/video_coding/codecs/interface/libvpx_interface.h" 27 #include "modules/video_coding/codecs/vp9/include/vp9.h" 28 #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" 29 #include "modules/video_coding/svc/scalable_video_controller.h" 30 #include "modules/video_coding/utility/framerate_controller_deprecated.h" 31 #include "rtc_base/containers/flat_map.h" 32 #include "rtc_base/experiments/encoder_info_settings.h" 33 #include "vpx/vp8cx.h" 34 35 namespace webrtc { 36 37 class LibvpxVp9Encoder : public VP9Encoder { 38 public: 39 LibvpxVp9Encoder(const cricket::VideoCodec& codec, 40 std::unique_ptr<LibvpxInterface> interface, 41 const FieldTrialsView& trials); 42 43 ~LibvpxVp9Encoder() override; 44 45 void SetFecControllerOverride( 46 FecControllerOverride* fec_controller_override) override; 47 48 int Release() override; 49 50 int InitEncode(const VideoCodec* codec_settings, 51 const Settings& settings) override; 52 53 int Encode(const VideoFrame& input_image, 54 const std::vector<VideoFrameType>* frame_types) override; 55 56 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; 57 58 void SetRates(const RateControlParameters& parameters) override; 59 60 EncoderInfo GetEncoderInfo() const override; 61 62 private: 63 // Determine number of encoder threads to use. 64 int NumberOfThreads(int width, int height, int number_of_cores); 65 66 // Call encoder initialize function and set control settings. 67 int InitAndSetControlSettings(const VideoCodec* inst); 68 69 bool PopulateCodecSpecific(CodecSpecificInfo* codec_specific, 70 absl::optional<int>* spatial_idx, 71 absl::optional<int>* temporal_idx, 72 const vpx_codec_cx_pkt& pkt); 73 void FillReferenceIndices(const vpx_codec_cx_pkt& pkt, 74 size_t pic_num, 75 bool inter_layer_predicted, 76 CodecSpecificInfoVP9* vp9_info); 77 void UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt, size_t pic_num); 78 vpx_svc_ref_frame_config_t SetReferences(bool is_key_pic, 79 int first_active_spatial_layer_id); 80 81 bool ExplicitlyConfiguredSpatialLayers() const; 82 bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation); 83 84 // Configures which spatial layers libvpx should encode according to 85 // configuration provided by svc_controller_. 86 void EnableSpatialLayer(int sid); 87 void DisableSpatialLayer(int sid); 88 void SetActiveSpatialLayers(); 89 90 void GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt); 91 92 // Callback function for outputting packets per spatial layer. 93 static void EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, 94 void* user_data); 95 96 void DeliverBufferedFrame(bool end_of_picture); 97 98 bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp); 99 100 // Determine maximum target for Intra frames 101 // 102 // Input: 103 // - optimal_buffer_size : Optimal buffer size 104 // Return Value : Max target size for Intra frames represented as 105 // percentage of the per frame bandwidth 106 uint32_t MaxIntraTarget(uint32_t optimal_buffer_size); 107 108 size_t SteadyStateSize(int sid, int tid); 109 110 void MaybeRewrapRawWithFormat(vpx_img_fmt fmt); 111 // Prepares `raw_` to reference image data of `buffer`, or of mapped or scaled 112 // versions of `buffer`. Returns the buffer that got referenced as a result, 113 // allowing the caller to keep a reference to it until after encoding has 114 // finished. On failure to convert the buffer, null is returned. 115 rtc::scoped_refptr<VideoFrameBuffer> PrepareBufferForProfile0( 116 rtc::scoped_refptr<VideoFrameBuffer> buffer); 117 118 const std::unique_ptr<LibvpxInterface> libvpx_; 119 EncodedImage encoded_image_; 120 CodecSpecificInfo codec_specific_; 121 EncodedImageCallback* encoded_complete_callback_; 122 VideoCodec codec_; 123 const VP9Profile profile_; 124 bool inited_; 125 int64_t timestamp_; 126 uint32_t rc_max_intra_target_; 127 vpx_codec_ctx_t* encoder_; 128 vpx_codec_enc_cfg_t* config_; 129 vpx_image_t* raw_; 130 vpx_svc_extra_cfg_t svc_params_; 131 const VideoFrame* input_image_; 132 GofInfoVP9 gof_; // Contains each frame's temporal information for 133 // non-flexible mode. 134 bool force_key_frame_; 135 size_t pics_since_key_; 136 uint8_t num_temporal_layers_; 137 uint8_t num_spatial_layers_; // Number of configured SLs 138 uint8_t num_active_spatial_layers_; // Number of actively encoded SLs 139 uint8_t first_active_layer_; 140 bool layer_deactivation_requires_key_frame_; 141 bool is_svc_; 142 InterLayerPredMode inter_layer_pred_; 143 bool external_ref_control_; 144 const bool trusted_rate_controller_; 145 bool layer_buffering_; 146 const bool full_superframe_drop_; 147 vpx_svc_frame_drop_t svc_drop_frame_; 148 bool first_frame_in_picture_; 149 VideoBitrateAllocation current_bitrate_allocation_; 150 bool ss_info_needed_; 151 bool force_all_active_layers_; 152 153 std::unique_ptr<ScalableVideoController> svc_controller_; 154 std::vector<FramerateControllerDeprecated> framerate_controller_; 155 156 // Used for flexible mode. 157 bool is_flexible_mode_; 158 struct RefFrameBuffer { 159 bool operator==(const RefFrameBuffer& o) { 160 return pic_num == o.pic_num && spatial_layer_id == o.spatial_layer_id && 161 temporal_layer_id == o.temporal_layer_id; 162 } 163 164 size_t pic_num = 0; 165 int spatial_layer_id = 0; 166 int temporal_layer_id = 0; 167 }; 168 std::array<RefFrameBuffer, kNumVp9Buffers> ref_buf_; 169 std::vector<ScalableVideoController::LayerFrameConfig> layer_frames_; 170 171 // Variable frame-rate related fields and methods. 172 const struct VariableFramerateExperiment { 173 bool enabled; 174 // Framerate is limited to this value in steady state. 175 float framerate_limit; 176 // This qp or below is considered a steady state. 177 int steady_state_qp; 178 // Frames of at least this percentage below ideal for configured bitrate are 179 // considered in a steady state. 180 int steady_state_undershoot_percentage; 181 // Number of consecutive frames with good QP and size required to detect 182 // the steady state. 183 int frames_before_steady_state; 184 } variable_framerate_experiment_; 185 static VariableFramerateExperiment ParseVariableFramerateConfig( 186 const FieldTrialsView& trials); 187 FramerateControllerDeprecated variable_framerate_controller_; 188 189 const struct QualityScalerExperiment { 190 int low_qp; 191 int high_qp; 192 bool enabled; 193 } quality_scaler_experiment_; 194 static QualityScalerExperiment ParseQualityScalerConfig( 195 const FieldTrialsView& trials); 196 const bool external_ref_ctrl_; 197 198 // Flags that can affect speed vs quality tradeoff, and are configureable per 199 // resolution ranges. 200 struct PerformanceFlags { 201 // If false, a lookup will be made in `settings_by_resolution` base on the 202 // highest currently active resolution, and the overall speed then set to 203 // to the `base_layer_speed` matching that entry. 204 // If true, each active resolution will have it's speed and deblock_mode set 205 // based on it resolution, and the high layer speed configured for non 206 // base temporal layer frames. 207 bool use_per_layer_speed = false; 208 209 struct ParameterSet { 210 int base_layer_speed = -1; // Speed setting for TL0. 211 int high_layer_speed = -1; // Speed setting for TL1-TL3. 212 // 0 = deblock all temporal layers (TL) 213 // 1 = disable deblock for top-most TL 214 // 2 = disable deblock for all TLs 215 int deblock_mode = 0; 216 bool allow_denoising = true; 217 }; 218 // Map from min pixel count to settings for that resolution and above. 219 // E.g. if you want some settings A if below wvga (640x360) and some other 220 // setting B at wvga and above, you'd use map {{0, A}, {230400, B}}. 221 flat_map<int, ParameterSet> settings_by_resolution; 222 }; 223 // Performance flags, ordered by `min_pixel_count`. 224 const PerformanceFlags performance_flags_; 225 // Caching of of `speed_configs_`, where index i maps to the resolution as 226 // specified in `codec_.spatialLayer[i]`. 227 std::vector<PerformanceFlags::ParameterSet> 228 performance_flags_by_spatial_index_; 229 void UpdatePerformanceFlags(); 230 static PerformanceFlags ParsePerformanceFlagsFromTrials( 231 const FieldTrialsView& trials); 232 static PerformanceFlags GetDefaultPerformanceFlags(); 233 234 int num_steady_state_frames_; 235 // Only set config when this flag is set. 236 bool config_changed_; 237 238 const LibvpxVp9EncoderInfoSettings encoder_info_override_; 239 }; 240 241 } // namespace webrtc 242 243 #endif // RTC_ENABLE_VP9 244 245 #endif // MODULES_VIDEO_CODING_CODECS_VP9_LIBVPX_VP9_ENCODER_H_ 246