1 /* 2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 12 #define VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 13 14 #include <map> 15 #include <memory> 16 #include <utility> 17 18 #include "absl/types/optional.h" 19 #include "api/video/video_stream_decoder.h" 20 #include "modules/video_coding/frame_buffer2.h" 21 #include "modules/video_coding/timing.h" 22 #include "rtc_base/platform_thread.h" 23 #include "rtc_base/synchronization/mutex.h" 24 #include "rtc_base/task_queue.h" 25 #include "rtc_base/thread_checker.h" 26 #include "system_wrappers/include/clock.h" 27 28 namespace webrtc { 29 30 class VideoStreamDecoderImpl : public VideoStreamDecoderInterface { 31 public: 32 VideoStreamDecoderImpl( 33 VideoStreamDecoderInterface::Callbacks* callbacks, 34 VideoDecoderFactory* decoder_factory, 35 TaskQueueFactory* task_queue_factory, 36 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings); 37 38 ~VideoStreamDecoderImpl() override; 39 40 void OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame) override; 41 42 void SetMinPlayoutDelay(TimeDelta min_delay) override; 43 void SetMaxPlayoutDelay(TimeDelta max_delay) override; 44 45 private: 46 class DecodeCallbacks : public DecodedImageCallback { 47 public: 48 explicit DecodeCallbacks(VideoStreamDecoderImpl* video_stream_decoder_impl); 49 int32_t Decoded(VideoFrame& decodedImage) override; 50 int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; 51 void Decoded(VideoFrame& decodedImage, 52 absl::optional<int32_t> decode_time_ms, 53 absl::optional<uint8_t> qp) override; 54 55 private: 56 VideoStreamDecoderImpl* const video_stream_decoder_impl_; 57 }; 58 59 enum DecodeResult { 60 kOk, 61 kOkRequestKeyframe, 62 kDecodeFailure, 63 }; 64 65 struct FrameTimestamps { 66 int64_t timestamp; 67 int64_t decode_start_time_ms; 68 int64_t render_time_us; 69 }; 70 71 void SaveFrameTimestamps(const video_coding::EncodedFrame& frame) 72 RTC_RUN_ON(bookkeeping_queue_); 73 FrameTimestamps* GetFrameTimestamps(int64_t timestamp) 74 RTC_RUN_ON(bookkeeping_queue_); 75 void StartNextDecode() RTC_RUN_ON(bookkeeping_queue_); 76 void OnNextFrameCallback(std::unique_ptr<video_coding::EncodedFrame> frame, 77 video_coding::FrameBuffer::ReturnReason res) 78 RTC_RUN_ON(bookkeeping_queue_); 79 void OnDecodedFrameCallback(VideoFrame& decodedImage, // NOLINT 80 absl::optional<int32_t> decode_time_ms, 81 absl::optional<uint8_t> qp); 82 83 VideoDecoder* GetDecoder(int payload_type) RTC_RUN_ON(decode_queue_); 84 VideoStreamDecoderImpl::DecodeResult DecodeFrame( 85 std::unique_ptr<video_coding::EncodedFrame> frame) 86 RTC_RUN_ON(decode_queue_); 87 88 VCMTiming timing_; 89 DecodeCallbacks decode_callbacks_; 90 91 // Some decoders are pipelined so it is not sufficient to save frame info 92 // for the last frame only. 93 static constexpr int kFrameTimestampsMemory = 8; 94 std::array<FrameTimestamps, kFrameTimestampsMemory> frame_timestamps_ 95 RTC_GUARDED_BY(bookkeeping_queue_); 96 int next_frame_timestamps_index_ RTC_GUARDED_BY(bookkeeping_queue_); 97 VideoStreamDecoderInterface::Callbacks* const callbacks_ 98 RTC_PT_GUARDED_BY(bookkeeping_queue_); 99 video_coding::VideoLayerFrameId last_continuous_id_ 100 RTC_GUARDED_BY(bookkeeping_queue_); 101 bool keyframe_required_ RTC_GUARDED_BY(bookkeeping_queue_); 102 103 absl::optional<int> current_payload_type_ RTC_GUARDED_BY(decode_queue_); 104 VideoDecoderFactory* const decoder_factory_ RTC_PT_GUARDED_BY(decode_queue_); 105 std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings_ 106 RTC_GUARDED_BY(decode_queue_); 107 108 // The |bookkeeping_queue_| use the |frame_buffer_| and also posts tasks to 109 // the |decode_queue_|. The |decode_queue_| in turn use the |decoder_| to 110 // decode frames. When the |decoder_| is done it will post back to the 111 // |bookkeeping_queue_| with the decoded frame. During shutdown we start by 112 // isolating the |bookkeeping_queue_| from the |decode_queue_|, so now it's 113 // safe for the |decode_queue_| to be destructed. After that the |decoder_| 114 // can be destructed, and then the |bookkeeping_queue_|. Finally the 115 // |frame_buffer_| can be destructed. 116 Mutex shut_down_mutex_; 117 bool shut_down_ RTC_GUARDED_BY(shut_down_mutex_); 118 video_coding::FrameBuffer frame_buffer_ RTC_GUARDED_BY(bookkeeping_queue_); 119 rtc::TaskQueue bookkeeping_queue_; 120 std::unique_ptr<VideoDecoder> decoder_ RTC_GUARDED_BY(decode_queue_); 121 rtc::TaskQueue decode_queue_; 122 }; 123 124 } // namespace webrtc 125 126 #endif // VIDEO_VIDEO_STREAM_DECODER_IMPL_H_ 127