1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 12 #define MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <utility> 18 #include <vector> 19 20 #include "absl/container/inlined_vector.h" 21 #include "api/video/encoded_frame.h" 22 #include "modules/video_coding/include/video_coding_defines.h" 23 #include "modules/video_coding/inter_frame_delay.h" 24 #include "modules/video_coding/jitter_estimator.h" 25 #include "modules/video_coding/utility/decoded_frames_history.h" 26 #include "rtc_base/constructor_magic.h" 27 #include "rtc_base/event.h" 28 #include "rtc_base/experiments/rtt_mult_experiment.h" 29 #include "rtc_base/numerics/sequence_number_util.h" 30 #include "rtc_base/synchronization/mutex.h" 31 #include "rtc_base/synchronization/sequence_checker.h" 32 #include "rtc_base/task_queue.h" 33 #include "rtc_base/task_utils/repeating_task.h" 34 #include "rtc_base/thread_annotations.h" 35 36 namespace webrtc { 37 38 class Clock; 39 class VCMReceiveStatisticsCallback; 40 class VCMJitterEstimator; 41 class VCMTiming; 42 43 namespace video_coding { 44 45 class FrameBuffer { 46 public: 47 enum ReturnReason { kFrameFound, kTimeout, kStopped }; 48 49 FrameBuffer(Clock* clock, 50 VCMTiming* timing, 51 VCMReceiveStatisticsCallback* stats_callback); 52 53 virtual ~FrameBuffer(); 54 55 // Insert a frame into the frame buffer. Returns the picture id 56 // of the last continuous frame or -1 if there is no continuous frame. 57 // TODO(philipel): Return a VideoLayerFrameId and not only the picture id. 58 int64_t InsertFrame(std::unique_ptr<EncodedFrame> frame); 59 60 // Get the next frame for decoding. Will return at latest after 61 // |max_wait_time_ms|. 62 void NextFrame( 63 int64_t max_wait_time_ms, 64 bool keyframe_required, 65 rtc::TaskQueue* callback_queue, 66 std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> handler); 67 68 // Tells the FrameBuffer which protection mode that is in use. Affects 69 // the frame timing. 70 // TODO(philipel): Remove this when new timing calculations has been 71 // implemented. 72 void SetProtectionMode(VCMVideoProtection mode); 73 74 // Start the frame buffer, has no effect if the frame buffer is started. 75 // The frame buffer is started upon construction. 76 void Start(); 77 78 // Stop the frame buffer, causing any sleeping thread in NextFrame to 79 // return immediately. 80 void Stop(); 81 82 // Updates the RTT for jitter buffer estimation. 83 void UpdateRtt(int64_t rtt_ms); 84 85 // Clears the FrameBuffer, removing all the buffered frames. 86 void Clear(); 87 88 private: 89 struct FrameInfo { 90 FrameInfo(); 91 FrameInfo(FrameInfo&&); 92 ~FrameInfo(); 93 94 // Which other frames that have direct unfulfilled dependencies 95 // on this frame. 96 absl::InlinedVector<VideoLayerFrameId, 8> dependent_frames; 97 98 // A frame is continiuous if it has all its referenced/indirectly 99 // referenced frames. 100 // 101 // How many unfulfilled frames this frame have until it becomes continuous. 102 size_t num_missing_continuous = 0; 103 104 // A frame is decodable if all its referenced frames have been decoded. 105 // 106 // How many unfulfilled frames this frame have until it becomes decodable. 107 size_t num_missing_decodable = 0; 108 109 // If this frame is continuous or not. 110 bool continuous = false; 111 112 // The actual EncodedFrame. 113 std::unique_ptr<EncodedFrame> frame; 114 }; 115 116 using FrameMap = std::map<VideoLayerFrameId, FrameInfo>; 117 118 // Check that the references of |frame| are valid. 119 bool ValidReferences(const EncodedFrame& frame) const; 120 121 int64_t FindNextFrame(int64_t now_ms) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 122 EncodedFrame* GetNextFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 123 124 void StartWaitForNextFrameOnQueue() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 125 void CancelCallback() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 126 127 // Update all directly dependent and indirectly dependent frames and mark 128 // them as continuous if all their references has been fulfilled. 129 void PropagateContinuity(FrameMap::iterator start) 130 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 131 132 // Marks the frame as decoded and updates all directly dependent frames. 133 void PropagateDecodability(const FrameInfo& info) 134 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 135 136 // Update the corresponding FrameInfo of |frame| and all FrameInfos that 137 // |frame| references. 138 // Return false if |frame| will never be decodable, true otherwise. 139 bool UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, 140 FrameMap::iterator info) 141 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 142 143 void UpdateJitterDelay() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 144 145 void UpdateTimingFrameInfo() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 146 147 void ClearFramesAndHistory() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 148 149 // Checks if the superframe, which current frame belongs to, is complete. 150 bool IsCompleteSuperFrame(const EncodedFrame& frame) 151 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 152 153 bool HasBadRenderTiming(const EncodedFrame& frame, int64_t now_ms) 154 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 155 156 // The cleaner solution would be to have the NextFrame function return a 157 // vector of frames, but until the decoding pipeline can support decoding 158 // multiple frames at the same time we combine all frames to one frame and 159 // return it. See bugs.webrtc.org/10064 160 EncodedFrame* CombineAndDeleteFrames( 161 const std::vector<EncodedFrame*>& frames) const; 162 163 SequenceChecker construction_checker_; 164 SequenceChecker callback_checker_; 165 166 // Stores only undecoded frames. 167 FrameMap frames_ RTC_GUARDED_BY(mutex_); 168 DecodedFramesHistory decoded_frames_history_ RTC_GUARDED_BY(mutex_); 169 170 Mutex mutex_; 171 Clock* const clock_; 172 173 rtc::TaskQueue* callback_queue_ RTC_GUARDED_BY(mutex_); 174 RepeatingTaskHandle callback_task_ RTC_GUARDED_BY(mutex_); 175 std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> 176 frame_handler_ RTC_GUARDED_BY(mutex_); 177 int64_t latest_return_time_ms_ RTC_GUARDED_BY(mutex_); 178 bool keyframe_required_ RTC_GUARDED_BY(mutex_); 179 180 VCMJitterEstimator jitter_estimator_ RTC_GUARDED_BY(mutex_); 181 VCMTiming* const timing_ RTC_GUARDED_BY(mutex_); 182 VCMInterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(mutex_); 183 absl::optional<VideoLayerFrameId> last_continuous_frame_ 184 RTC_GUARDED_BY(mutex_); 185 std::vector<FrameMap::iterator> frames_to_decode_ RTC_GUARDED_BY(mutex_); 186 bool stopped_ RTC_GUARDED_BY(mutex_); 187 VCMVideoProtection protection_mode_ RTC_GUARDED_BY(mutex_); 188 VCMReceiveStatisticsCallback* const stats_callback_; 189 int64_t last_log_non_decoded_ms_ RTC_GUARDED_BY(mutex_); 190 191 const bool add_rtt_to_playout_delay_; 192 193 // rtt_mult experiment settings. 194 const absl::optional<RttMultExperiment::Settings> rtt_mult_settings_; 195 196 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer); 197 }; 198 199 } // namespace video_coding 200 } // namespace webrtc 201 202 #endif // MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 203