1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ 12 #define API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ 13 14 #include <array> 15 #include <memory> 16 #include <vector> 17 18 #include "absl/types/optional.h" 19 #include "api/fec_controller_override.h" 20 #include "api/video_codecs/video_codec.h" 21 #include "api/video_codecs/video_encoder.h" 22 #include "api/video_codecs/vp8_frame_config.h" 23 24 namespace webrtc { 25 26 // Some notes on the prerequisites of the TemporalLayers interface. 27 // * Vp8FrameBufferController is not thread safe, synchronization is the 28 // caller's responsibility. 29 // * The encoder is assumed to encode all frames in order, and callbacks to 30 // PopulateCodecSpecific() / OnEncodeDone() must happen in the same order. 31 // 32 // This means that in the case of pipelining encoders, it is OK to have a chain 33 // of calls such as this: 34 // - NextFrameConfig(timestampA) 35 // - NextFrameConfig(timestampB) 36 // - PopulateCodecSpecific(timestampA, ...) 37 // - NextFrameConfig(timestampC) 38 // - OnEncodeDone(timestampA, 1234, ...) 39 // - NextFrameConfig(timestampC) 40 // - OnEncodeDone(timestampB, 0, ...) 41 // - OnEncodeDone(timestampC, 1234, ...) 42 // Note that NextFrameConfig() for a new frame can happen before 43 // OnEncodeDone() for a previous one, but calls themselves must be both 44 // synchronized (e.g. run on a task queue) and in order (per type). 45 // 46 // TODO(eladalon): Revise comment (referring to PopulateCodecSpecific in this 47 // context is not very meaningful). 48 49 struct CodecSpecificInfo; 50 51 // Each member represents an override of the VPX configuration if the optional 52 // value is set. 53 struct Vp8EncoderConfig { 54 struct TemporalLayerConfig { 55 bool operator!=(const TemporalLayerConfig& other) const { 56 return ts_number_layers != other.ts_number_layers || 57 ts_target_bitrate != other.ts_target_bitrate || 58 ts_rate_decimator != other.ts_rate_decimator || 59 ts_periodicity != other.ts_periodicity || 60 ts_layer_id != other.ts_layer_id; 61 } 62 63 static constexpr size_t kMaxPeriodicity = 16; 64 static constexpr size_t kMaxLayers = 5; 65 66 // Number of active temporal layers. Set to 0 if not used. 67 uint32_t ts_number_layers; 68 69 // Arrays of length |ts_number_layers|, indicating (cumulative) target 70 // bitrate and rate decimator (e.g. 4 if every 4th frame is in the given 71 // layer) for each active temporal layer, starting with temporal id 0. 72 std::array<uint32_t, kMaxLayers> ts_target_bitrate; 73 std::array<uint32_t, kMaxLayers> ts_rate_decimator; 74 75 // The periodicity of the temporal pattern. Set to 0 if not used. 76 uint32_t ts_periodicity; 77 78 // Array of length |ts_periodicity| indicating the sequence of temporal id's 79 // to assign to incoming frames. 80 std::array<uint32_t, kMaxPeriodicity> ts_layer_id; 81 }; 82 83 absl::optional<TemporalLayerConfig> temporal_layer_config; 84 85 // Target bitrate, in bps. 86 absl::optional<uint32_t> rc_target_bitrate; 87 88 // Clamp QP to max. Use 0 to disable clamping. 89 absl::optional<uint32_t> rc_max_quantizer; 90 91 // Error resilience mode. 92 absl::optional<uint32_t> g_error_resilient; 93 94 // If set to true, all previous configuration overrides should be reset. 95 bool reset_previous_configuration_overrides = false; 96 }; 97 98 // This interface defines a way of delegating the logic of buffer management. 99 // Multiple streams may be controlled by a single controller, demuxing between 100 // them using stream_index. 101 class Vp8FrameBufferController { 102 public: 103 virtual ~Vp8FrameBufferController() = default; 104 105 // Set limits on QP. 106 // The limits are suggestion-only; the controller is allowed to exceed them. 107 virtual void SetQpLimits(size_t stream_index, int min_qp, int max_qp) = 0; 108 109 // Number of streamed controlled by |this|. 110 virtual size_t StreamCount() const = 0; 111 112 // If this method returns true, the encoder is free to drop frames for 113 // instance in an effort to uphold encoding bitrate. 114 // If this return false, the encoder must not drop any frames unless: 115 // 1. Requested to do so via Vp8FrameConfig.drop_frame 116 // 2. The frame to be encoded is requested to be a keyframe 117 // 3. The encoder detected a large overshoot and decided to drop and then 118 // re-encode the image at a low bitrate. In this case the encoder should 119 // call OnFrameDropped() once to indicate drop, and then call 120 // OnEncodeDone() again when the frame has actually been encoded. 121 virtual bool SupportsEncoderFrameDropping(size_t stream_index) const = 0; 122 123 // New target bitrate for a stream (each entry in 124 // |bitrates_bps| is for another temporal layer). 125 virtual void OnRatesUpdated(size_t stream_index, 126 const std::vector<uint32_t>& bitrates_bps, 127 int framerate_fps) = 0; 128 129 // Called by the encoder before encoding a frame. Returns a set of overrides 130 // the controller wishes to enact in the encoder's configuration. 131 // If a value is not overridden, previous overrides are still in effect. 132 // However, if |Vp8EncoderConfig::reset_previous_configuration_overrides| 133 // is set to |true|, all previous overrides are reset. 134 virtual Vp8EncoderConfig UpdateConfiguration(size_t stream_index) = 0; 135 136 // Returns the recommended VP8 encode flags needed. 137 // The timestamp may be used as both a time and a unique identifier, and so 138 // the caller must make sure no two frames use the same timestamp. 139 // The timestamp uses a 90kHz RTP clock. 140 // After calling this method, first call the actual encoder with the provided 141 // frame configuration, and then OnEncodeDone() below. 142 virtual Vp8FrameConfig NextFrameConfig(size_t stream_index, 143 uint32_t rtp_timestamp) = 0; 144 145 // Called after the encode step is done. |rtp_timestamp| must match the 146 // parameter use in the NextFrameConfig() call. 147 // |is_keyframe| must be true iff the encoder decided to encode this frame as 148 // a keyframe. 149 // If |info| is not null, the encoder may update |info| with codec specific 150 // data such as temporal id. |qp| should indicate the frame-level QP this 151 // frame was encoded at. If the encoder does not support extracting this, |qp| 152 // should be set to 0. 153 virtual void OnEncodeDone(size_t stream_index, 154 uint32_t rtp_timestamp, 155 size_t size_bytes, 156 bool is_keyframe, 157 int qp, 158 CodecSpecificInfo* info) = 0; 159 160 // Called when a frame is dropped by the encoder. 161 virtual void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) = 0; 162 163 // Called by the encoder when the packet loss rate changes. 164 // |packet_loss_rate| runs between 0.0 (no loss) and 1.0 (everything lost). 165 virtual void OnPacketLossRateUpdate(float packet_loss_rate) = 0; 166 167 // Called by the encoder when the round trip time changes. 168 virtual void OnRttUpdate(int64_t rtt_ms) = 0; 169 170 // Called when a loss notification is received. 171 virtual void OnLossNotification( 172 const VideoEncoder::LossNotification& loss_notification) = 0; 173 }; 174 175 // Interface for a factory of Vp8FrameBufferController instances. 176 class Vp8FrameBufferControllerFactory { 177 public: 178 virtual ~Vp8FrameBufferControllerFactory() = default; 179 180 // Clones oneself. (Avoids Vp8FrameBufferControllerFactoryFactory.) 181 virtual std::unique_ptr<Vp8FrameBufferControllerFactory> Clone() const = 0; 182 183 // Create a Vp8FrameBufferController instance. 184 virtual std::unique_ptr<Vp8FrameBufferController> Create( 185 const VideoCodec& codec, 186 const VideoEncoder::Settings& settings, 187 FecControllerOverride* fec_controller_override) = 0; 188 }; 189 190 } // namespace webrtc 191 192 #endif // API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ 193