1 /* 2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 12 #define VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 13 14 #include <map> 15 #include <memory> 16 #include <utility> 17 18 #include "absl/types/optional.h" 19 #include "api/field_trials_view.h" 20 #include "api/sequence_checker.h" 21 #include "api/transport/field_trial_based_config.h" 22 #include "api/video/video_stream_decoder.h" 23 #include "modules/video_coding/frame_buffer2.h" 24 #include "modules/video_coding/timing/timing.h" 25 #include "rtc_base/memory/always_valid_pointer.h" 26 #include "rtc_base/platform_thread.h" 27 #include "rtc_base/synchronization/mutex.h" 28 #include "rtc_base/task_queue.h" 29 #include "system_wrappers/include/clock.h" 30 31 namespace webrtc { 32 33 class VideoStreamDecoderImpl : public VideoStreamDecoderInterface { 34 public: 35 VideoStreamDecoderImpl( 36 VideoStreamDecoderInterface::Callbacks* callbacks, 37 VideoDecoderFactory* decoder_factory, 38 TaskQueueFactory* task_queue_factory, 39 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings, 40 const FieldTrialsView* field_trials); 41 42 ~VideoStreamDecoderImpl() override; 43 44 void OnFrame(std::unique_ptr<EncodedFrame> frame) override; 45 46 void SetMinPlayoutDelay(TimeDelta min_delay) override; 47 void SetMaxPlayoutDelay(TimeDelta max_delay) override; 48 49 private: 50 class DecodeCallbacks : public DecodedImageCallback { 51 public: 52 explicit DecodeCallbacks(VideoStreamDecoderImpl* video_stream_decoder_impl); 53 int32_t Decoded(VideoFrame& decodedImage) override; 54 int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; 55 void Decoded(VideoFrame& decodedImage, 56 absl::optional<int32_t> decode_time_ms, 57 absl::optional<uint8_t> qp) override; 58 59 private: 60 VideoStreamDecoderImpl* const video_stream_decoder_impl_; 61 }; 62 63 enum DecodeResult { 64 kOk, 65 kOkRequestKeyframe, 66 kDecodeFailure, 67 }; 68 69 struct FrameInfo { 70 int64_t timestamp = -1; 71 int64_t decode_start_time_ms; 72 int64_t render_time_us; 73 VideoContentType content_type; 74 }; 75 76 void SaveFrameInfo(const EncodedFrame& frame) RTC_RUN_ON(bookkeeping_queue_); 77 FrameInfo* GetFrameInfo(int64_t timestamp) RTC_RUN_ON(bookkeeping_queue_); 78 void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_); 79 void OnNextFrameCallback(std::unique_ptr<EncodedFrame> frame) 80 RTC_RUN_ON(bookkeeping_queue_); 81 void OnDecodedFrameCallback(VideoFrame& decodedImage, // NOLINT 82 absl::optional<int32_t> decode_time_ms, 83 absl::optional<uint8_t> qp); 84 85 VideoDecoder* GetDecoder(int payload_type) RTC_RUN_ON(decode_queue_); 86 VideoStreamDecoderImpl::DecodeResult DecodeFrame( 87 std::unique_ptr<EncodedFrame> frame) RTC_RUN_ON(decode_queue_); 88 89 AlwaysValidPointer<const FieldTrialsView, FieldTrialBasedConfig> 90 field_trials_; 91 VCMTiming timing_; 92 DecodeCallbacks decode_callbacks_; 93 94 // Some decoders are pipelined so it is not sufficient to save frame info 95 // for the last frame only. 96 static constexpr int kFrameInfoMemory = 8; 97 std::array<FrameInfo, kFrameInfoMemory> frame_info_ 98 RTC_GUARDED_BY(bookkeeping_queue_); 99 int next_frame_info_index_ RTC_GUARDED_BY(bookkeeping_queue_); 100 VideoStreamDecoderInterface::Callbacks* const callbacks_ 101 RTC_PT_GUARDED_BY(bookkeeping_queue_); 102 int64_t last_continuous_frame_id_ RTC_GUARDED_BY(bookkeeping_queue_) = -1; 103 bool keyframe_required_ RTC_GUARDED_BY(bookkeeping_queue_); 104 105 absl::optional<int> current_payload_type_ RTC_GUARDED_BY(decode_queue_); 106 VideoDecoderFactory* const decoder_factory_ RTC_PT_GUARDED_BY(decode_queue_); 107 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings_ 108 RTC_GUARDED_BY(decode_queue_); 109 110 // The `bookkeeping_queue_` use the `frame_buffer_` and also posts tasks to 111 // the `decode_queue_`. The `decode_queue_` in turn use the `decoder_` to 112 // decode frames. When the `decoder_` is done it will post back to the 113 // `bookkeeping_queue_` with the decoded frame. During shutdown we start by 114 // isolating the `bookkeeping_queue_` from the `decode_queue_`, so now it's 115 // safe for the `decode_queue_` to be destructed. After that the `decoder_` 116 // can be destructed, and then the `bookkeeping_queue_`. Finally the 117 // `frame_buffer_` can be destructed. 118 Mutex shut_down_mutex_; 119 bool shut_down_ RTC_GUARDED_BY(shut_down_mutex_); 120 video_coding::FrameBuffer frame_buffer_ RTC_GUARDED_BY(bookkeeping_queue_); 121 rtc::TaskQueue bookkeeping_queue_; 122 std::unique_ptr<VideoDecoder> decoder_ RTC_GUARDED_BY(decode_queue_); 123 rtc::TaskQueue decode_queue_; 124 }; 125 126 } // namespace webrtc 127 128 #endif // VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 129