1 // Copyright 2021 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef CAST_STANDALONE_SENDER_STREAMING_VIDEO_ENCODER_H_ 6 #define CAST_STANDALONE_SENDER_STREAMING_VIDEO_ENCODER_H_ 7 8 #include <algorithm> 9 #include <condition_variable> // NOLINT 10 #include <functional> 11 #include <memory> 12 #include <mutex> 13 #include <queue> 14 #include <thread> 15 #include <vector> 16 17 #include "absl/base/thread_annotations.h" 18 #include "cast/streaming/constants.h" 19 #include "cast/streaming/frame_id.h" 20 #include "cast/streaming/rtp_time.h" 21 #include "platform/api/task_runner.h" 22 #include "platform/api/time.h" 23 24 namespace openscreen { 25 26 class TaskRunner; 27 28 namespace cast { 29 30 class Sender; 31 32 class StreamingVideoEncoder { 33 public: 34 // Configurable parameters passed to the StreamingVpxEncoder constructor. 35 struct Parameters { 36 // Number of threads to parallelize frame encoding. This should be set based 37 // on the number of CPU cores available for encoding, but no more than 8. 38 int num_encode_threads = 39 std::min(std::max<int>(std::thread::hardware_concurrency(), 1), 8); 40 41 // Best-quality quantizer (lower is better quality). Range: [0,63] 42 int min_quantizer = 4; 43 44 // Worst-quality quantizer (lower is better quality). Range: [0,63] 45 int max_quantizer = kMaxQuantizer; 46 47 // Worst-quality quantizer to use when the CPU is extremely constrained. 48 // Range: [min_quantizer,max_quantizer] 49 int max_cpu_saver_quantizer = 25; 50 51 // Maximum amount of wall-time a frame's encode can take, relative to the 52 // frame's duration, before the CPU-saver logic is activated. The default 53 // (70%) is appropriate for systems with four or more cores, but should be 54 // reduced (e.g., 50%) for systems with fewer than three cores. 55 // 56 // Example: For 30 FPS (continuous) video, the frame duration is ~33.3ms, 57 // and a value of 0.5 here would mean that the CPU-saver logic starts 58 // sacrificing quality when frame encodes start taking longer than ~16.7ms. 59 double max_time_utilization = 0.7; 60 61 // Determines which codec (VP8, VP9, or AV1) is to be used for encoding. 62 // Defaults to VP8. 63 VideoCodec codec = VideoCodec::kVp8; 64 }; 65 66 // Represents an input VideoFrame, passed to EncodeAndSend(). 67 struct VideoFrame { 68 // Image width and height. 69 int width = 0; 70 int height = 0; 71 72 // I420 format image pointers and row strides (the number of bytes between 73 // the start of successive rows). The pointers only need to remain valid 74 // until the EncodeAndSend() call returns. 75 const uint8_t* yuv_planes[3] = {}; 76 int yuv_strides[3] = {}; 77 78 // How long this frame will be held before the next frame will be displayed, 79 // or zero if unknown. The frame duration is passed to the video codec, 80 // affecting a number of important behaviors, including: per-frame 81 // bandwidth, CPU time spent encoding, temporal quality trade-offs, and 82 // key/golden/alt-ref frame generation intervals. 83 Clock::duration duration; 84 }; 85 86 // Performance statistics for a single frame's encode. 87 // 88 // For full details on how to use these stats in an end-to-end system, see: 89 // https://www.chromium.org/developers/design-documents/ 90 // auto-throttled-screen-capture-and-mirroring 91 // and https://source.chromium.org/chromium/chromium/src/+/master: 92 // media/cast/sender/performance_metrics_overlay.h 93 struct Stats { 94 // The Cast Streaming ID that was assigned to the frame. 95 FrameId frame_id; 96 97 // The RTP timestamp of the frame. 98 RtpTimeTicks rtp_timestamp; 99 100 // How long the frame took to encode. This is wall time, not CPU time or 101 // some other load metric. 102 Clock::duration encode_wall_time; 103 104 // The frame's predicted duration; or, the actual duration if it was 105 // provided in the VideoFrame. 106 Clock::duration frame_duration; 107 108 // The encoded frame's size in bytes. 109 int encoded_size = 0; 110 111 // The average size of an encoded frame in bytes, having this 112 // |frame_duration| and current target bitrate. 113 double target_size = 0.0; 114 115 // The actual quantizer the video encoder used, in the range [0,63]. 116 int quantizer = 0; 117 118 // The "hindsight" quantizer value that would have produced the best quality 119 // encoding of the frame at the current target bitrate. The nominal range is 120 // [0.0,63.0]. If it is larger than 63.0, then it was impossible to 121 // encode the frame within the current target bitrate (e.g., too much 122 // "entropy" in the image, or too low a target bitrate). 123 double perfect_quantizer = 0.0; 124 125 // Utilization feedback metrics. The nominal range for each of these is 126 // [0.0,1.0] where 1.0 means "the entire budget available for the frame was 127 // exhausted." Going above 1.0 is okay for one or a few frames, since it's 128 // the average over many frames that matters before the system is considered 129 // "redlining." 130 // 131 // The max of these three provides an overall utilization control signal. 132 // The usual approach is for upstream control logic to increase/decrease the 133 // data volume (e.g., video resolution and/or frame rate) to maintain a good 134 // target point. time_utilizationStats135 double time_utilization() const { 136 return static_cast<double>(encode_wall_time.count()) / 137 frame_duration.count(); 138 } space_utilizationStats139 double space_utilization() const { return encoded_size / target_size; } entropy_utilizationStats140 double entropy_utilization() const { 141 return perfect_quantizer / kMaxQuantizer; 142 } 143 }; 144 145 virtual ~StreamingVideoEncoder(); 146 147 // Get/Set the target bitrate. This may be changed at any time, as frequently 148 // as desired, and it will take effect internally as soon as possible. 149 virtual int GetTargetBitrate() const = 0; 150 virtual void SetTargetBitrate(int new_bitrate) = 0; 151 152 // Encode |frame| using the video encoder, assemble an EncodedFrame, and 153 // enqueue into the Sender. The frame may be dropped if too many frames are 154 // in-flight. If provided, the |stats_callback| is run after the frame is 155 // enqueued in the Sender (via the main TaskRunner). 156 virtual void EncodeAndSend(const VideoFrame& frame, 157 Clock::time_point reference_time, 158 std::function<void(Stats)> stats_callback) = 0; 159 160 static constexpr int kMinQuantizer = 0; 161 static constexpr int kMaxQuantizer = 63; 162 163 protected: 164 StreamingVideoEncoder(const Parameters& params, 165 TaskRunner* task_runner, 166 Sender* sender); 167 168 // This is the equivalent change in encoding speed per one quantizer step. 169 static constexpr double kEquivalentEncodingSpeedStepPerQuantizerStep = 170 1 / 20.0; 171 172 // Updates the |ideal_speed_setting_|, to take effect with the next frame 173 // encode, based on the given performance |stats|. 174 void UpdateSpeedSettingForNextFrame(const Stats& stats); 175 176 const Parameters params_; 177 TaskRunner* const main_task_runner_; 178 Sender* const sender_; 179 180 // These represent the magnitude of the AV1 speed setting, where larger values 181 // (i.e., faster speed) request less CPU usage but will provide lower video 182 // quality. Only the encode thread accesses these. 183 double ideal_speed_setting_; // A time-weighted average, from measurements. 184 int current_speed_setting_; // Current |encoder_| speed setting. 185 186 // This member should be last in the class since the thread should not start 187 // until all above members have been initialized by the constructor. 188 std::thread encode_thread_; 189 }; 190 191 } // namespace cast 192 } // namespace openscreen 193 194 #endif // CAST_STANDALONE_SENDER_STREAMING_VIDEO_ENCODER_H_ 195