1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 12 #define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/strings/string_view.h" 20 #include "api/test/video_quality_analyzer_interface.h" 21 #include "api/video/encoded_image.h" 22 #include "api/video/video_frame.h" 23 #include "api/video_codecs/sdp_video_format.h" 24 #include "api/video_codecs/video_decoder.h" 25 #include "api/video_codecs/video_decoder_factory.h" 26 #include "rtc_base/synchronization/mutex.h" 27 #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" 28 #include "test/pc/e2e/analyzer/video/id_generator.h" 29 30 namespace webrtc { 31 namespace webrtc_pc_e2e { 32 33 // QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject 34 // VideoQualityAnalyzerInterface before and after decoder. 35 // 36 // QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. 37 // It registers its own DecodedImageCallback in the origin decoder and will 38 // store user specified callback inside itself. 39 // 40 // When Decode(...) will be invoked, quality decoder first will extract frame id 41 // from passed EncodedImage with EncodedImageIdExtracor that was specified in 42 // constructor, then will call video quality analyzer, with correct 43 // EncodedImage and only then will pass image to origin decoder. 44 // 45 // When origin decoder decodes the image it will call quality decoder's special 46 // callback, where video analyzer will be called again and then decoded frame 47 // will be passed to origin callback, provided by user. 48 // 49 // Quality decoder registers its own callback in origin decoder, at the same 50 // time the user registers their callback in quality decoder. 51 class QualityAnalyzingVideoDecoder : public VideoDecoder { 52 public: 53 // Creates analyzing decoder. |id| is unique coding entity id, that will 54 // be used to distinguish all encoders and decoders inside 55 // EncodedImageDataInjector and EncodedImageIdExtracor. 56 QualityAnalyzingVideoDecoder(int id, 57 absl::string_view peer_name, 58 std::unique_ptr<VideoDecoder> delegate, 59 EncodedImageDataExtractor* extractor, 60 VideoQualityAnalyzerInterface* analyzer); 61 ~QualityAnalyzingVideoDecoder() override; 62 63 // Methods of VideoDecoder interface. 64 int32_t InitDecode(const VideoCodec* codec_settings, 65 int32_t number_of_cores) override; 66 int32_t Decode(const EncodedImage& input_image, 67 bool missing_frames, 68 int64_t render_time_ms) override; 69 int32_t RegisterDecodeCompleteCallback( 70 DecodedImageCallback* callback) override; 71 int32_t Release() override; 72 bool PrefersLateDecoding() const override; 73 const char* ImplementationName() const override; 74 75 private: 76 class DecoderCallback : public DecodedImageCallback { 77 public: 78 explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); 79 ~DecoderCallback() override; 80 81 void SetDelegateCallback(DecodedImageCallback* delegate); 82 83 // Methods of DecodedImageCallback interface. 84 int32_t Decoded(VideoFrame& decodedImage) override; 85 int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; 86 void Decoded(VideoFrame& decodedImage, 87 absl::optional<int32_t> decode_time_ms, 88 absl::optional<uint8_t> qp) override; 89 90 int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, 91 uint32_t timestamp_ms); 92 93 private: 94 rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer(); 95 96 QualityAnalyzingVideoDecoder* const decoder_; 97 98 rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_; 99 100 Mutex callback_lock_; 101 DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_lock_); 102 }; 103 104 void OnFrameDecoded(VideoFrame* frame, 105 absl::optional<int32_t> decode_time_ms, 106 absl::optional<uint8_t> qp); 107 108 const int id_; 109 const std::string peer_name_; 110 const std::string implementation_name_; 111 std::unique_ptr<VideoDecoder> delegate_; 112 EncodedImageDataExtractor* const extractor_; 113 VideoQualityAnalyzerInterface* const analyzer_; 114 std::unique_ptr<DecoderCallback> analyzing_callback_; 115 116 // VideoDecoder interface assumes async delivery of decoded video frames. 117 // This lock is used to protect shared state, that have to be propagated 118 // from received EncodedImage to resulted VideoFrame. 119 Mutex lock_; 120 121 std::map<uint32_t, uint16_t> timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); 122 // Stores currently being decoded images by frame id. Because 123 // EncodedImageDataExtractor can create new copy on EncodedImage we need to 124 // ensure, that this image won't be deleted during async decoding. To do it 125 // all images are putted into this map and removed from here inside callback. 126 std::map<uint16_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(lock_); 127 }; 128 129 // Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by 130 // specified factory as delegates. Forwards all other calls to specified 131 // factory. 132 class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { 133 public: 134 QualityAnalyzingVideoDecoderFactory( 135 absl::string_view peer_name, 136 std::unique_ptr<VideoDecoderFactory> delegate, 137 IdGenerator<int>* id_generator, 138 EncodedImageDataExtractor* extractor, 139 VideoQualityAnalyzerInterface* analyzer); 140 ~QualityAnalyzingVideoDecoderFactory() override; 141 142 // Methods of VideoDecoderFactory interface. 143 std::vector<SdpVideoFormat> GetSupportedFormats() const override; 144 std::unique_ptr<VideoDecoder> CreateVideoDecoder( 145 const SdpVideoFormat& format) override; 146 std::unique_ptr<VideoDecoder> LegacyCreateVideoDecoder( 147 const SdpVideoFormat& format, 148 const std::string& receive_stream_id) override; 149 150 private: 151 const std::string peer_name_; 152 std::unique_ptr<VideoDecoderFactory> delegate_; 153 IdGenerator<int>* const id_generator_; 154 EncodedImageDataExtractor* const extractor_; 155 VideoQualityAnalyzerInterface* const analyzer_; 156 }; 157 158 } // namespace webrtc_pc_e2e 159 } // namespace webrtc 160 161 #endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 162