1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 12 #define MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <utility> 18 #include <vector> 19 20 #include "absl/container/inlined_vector.h" 21 #include "api/field_trials_view.h" 22 #include "api/sequence_checker.h" 23 #include "api/task_queue/task_queue_base.h" 24 #include "api/video/encoded_frame.h" 25 #include "modules/video_coding/include/video_coding_defines.h" 26 #include "modules/video_coding/timing/inter_frame_delay.h" 27 #include "modules/video_coding/timing/jitter_estimator.h" 28 #include "modules/video_coding/utility/decoded_frames_history.h" 29 #include "rtc_base/event.h" 30 #include "rtc_base/experiments/field_trial_parser.h" 31 #include "rtc_base/experiments/rtt_mult_experiment.h" 32 #include "rtc_base/numerics/sequence_number_util.h" 33 #include "rtc_base/synchronization/mutex.h" 34 #include "rtc_base/system/no_unique_address.h" 35 #include "rtc_base/task_utils/repeating_task.h" 36 #include "rtc_base/thread_annotations.h" 37 38 namespace webrtc { 39 40 class Clock; 41 class VCMReceiveStatisticsCallback; 42 class JitterEstimator; 43 class VCMTiming; 44 45 namespace video_coding { 46 47 class FrameBuffer { 48 public: 49 FrameBuffer(Clock* clock, 50 VCMTiming* timing, 51 const FieldTrialsView& field_trials); 52 53 FrameBuffer() = delete; 54 FrameBuffer(const FrameBuffer&) = delete; 55 FrameBuffer& operator=(const FrameBuffer&) = delete; 56 57 virtual ~FrameBuffer(); 58 59 // Insert a frame into the frame buffer. Returns the picture id 60 // of the last continuous frame or -1 if there is no continuous frame. 61 int64_t InsertFrame(std::unique_ptr<EncodedFrame> frame); 62 63 using NextFrameCallback = std::function<void(std::unique_ptr<EncodedFrame>)>; 64 // Get the next frame for decoding. `handler` is invoked with the next frame 65 // or with nullptr if no frame is ready for decoding after `max_wait_time_ms`. 66 void NextFrame(int64_t max_wait_time_ms, 67 bool keyframe_required, 68 TaskQueueBase* callback_queue, 69 NextFrameCallback handler); 70 71 // Tells the FrameBuffer which protection mode that is in use. Affects 72 // the frame timing. 73 // TODO(philipel): Remove this when new timing calculations has been 74 // implemented. 75 void SetProtectionMode(VCMVideoProtection mode); 76 77 // Stop the frame buffer, causing any sleeping thread in NextFrame to 78 // return immediately. 79 void Stop(); 80 81 // Updates the RTT for jitter buffer estimation. 82 void UpdateRtt(int64_t rtt_ms); 83 84 // Clears the FrameBuffer, removing all the buffered frames. 85 void Clear(); 86 87 int Size(); 88 89 private: 90 struct FrameInfo { 91 FrameInfo(); 92 FrameInfo(FrameInfo&&); 93 ~FrameInfo(); 94 95 // Which other frames that have direct unfulfilled dependencies 96 // on this frame. 97 absl::InlinedVector<int64_t, 8> dependent_frames; 98 99 // A frame is continiuous if it has all its referenced/indirectly 100 // referenced frames. 101 // 102 // How many unfulfilled frames this frame have until it becomes continuous. 103 size_t num_missing_continuous = 0; 104 105 // A frame is decodable if all its referenced frames have been decoded. 106 // 107 // How many unfulfilled frames this frame have until it becomes decodable. 108 size_t num_missing_decodable = 0; 109 110 // If this frame is continuous or not. 111 bool continuous = false; 112 113 // The actual EncodedFrame. 114 std::unique_ptr<EncodedFrame> frame; 115 }; 116 117 using FrameMap = std::map<int64_t, FrameInfo>; 118 119 // Check that the references of `frame` are valid. 120 bool ValidReferences(const EncodedFrame& frame) const; 121 122 int64_t FindNextFrame(Timestamp now) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 123 std::unique_ptr<EncodedFrame> GetNextFrame() 124 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 125 126 void StartWaitForNextFrameOnQueue() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 127 void CancelCallback() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 128 129 // Update all directly dependent and indirectly dependent frames and mark 130 // them as continuous if all their references has been fulfilled. 131 void PropagateContinuity(FrameMap::iterator start) 132 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 133 134 // Marks the frame as decoded and updates all directly dependent frames. 135 void PropagateDecodability(const FrameInfo& info) 136 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 137 138 // Update the corresponding FrameInfo of `frame` and all FrameInfos that 139 // `frame` references. 140 // Return false if `frame` will never be decodable, true otherwise. 141 bool UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, 142 FrameMap::iterator info) 143 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 144 145 void ClearFramesAndHistory() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 146 147 // The cleaner solution would be to have the NextFrame function return a 148 // vector of frames, but until the decoding pipeline can support decoding 149 // multiple frames at the same time we combine all frames to one frame and 150 // return it. See bugs.webrtc.org/10064 151 std::unique_ptr<EncodedFrame> CombineAndDeleteFrames( 152 std::vector<std::unique_ptr<EncodedFrame>> frames) const; 153 154 RTC_NO_UNIQUE_ADDRESS SequenceChecker construction_checker_; 155 RTC_NO_UNIQUE_ADDRESS SequenceChecker callback_checker_; 156 157 // Stores only undecoded frames. 158 FrameMap frames_ RTC_GUARDED_BY(mutex_); 159 DecodedFramesHistory decoded_frames_history_ RTC_GUARDED_BY(mutex_); 160 161 Mutex mutex_; 162 Clock* const clock_; 163 164 TaskQueueBase* callback_queue_ RTC_GUARDED_BY(mutex_); 165 RepeatingTaskHandle callback_task_ RTC_GUARDED_BY(mutex_); 166 NextFrameCallback frame_handler_ RTC_GUARDED_BY(mutex_); 167 int64_t latest_return_time_ms_ RTC_GUARDED_BY(mutex_); 168 bool keyframe_required_ RTC_GUARDED_BY(mutex_); 169 170 JitterEstimator jitter_estimator_ RTC_GUARDED_BY(mutex_); 171 VCMTiming* const timing_ RTC_GUARDED_BY(mutex_); 172 InterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(mutex_); 173 absl::optional<int64_t> last_continuous_frame_ RTC_GUARDED_BY(mutex_); 174 std::vector<FrameMap::iterator> frames_to_decode_ RTC_GUARDED_BY(mutex_); 175 bool stopped_ RTC_GUARDED_BY(mutex_); 176 VCMVideoProtection protection_mode_ RTC_GUARDED_BY(mutex_); 177 int64_t last_log_non_decoded_ms_ RTC_GUARDED_BY(mutex_); 178 179 // rtt_mult experiment settings. 180 const absl::optional<RttMultExperiment::Settings> rtt_mult_settings_; 181 182 // Maximum number of frames in the decode queue to allow pacing. If the 183 // queue grows beyond the max limit, pacing will be disabled and frames will 184 // be pushed to the decoder as soon as possible. This only has an effect 185 // when the low-latency rendering path is active, which is indicated by 186 // the frame's render time == 0. 187 FieldTrialParameter<unsigned> zero_playout_delay_max_decode_queue_size_; 188 }; 189 190 } // namespace video_coding 191 } // namespace webrtc 192 193 #endif // MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 194