1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 17 #define MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 18 19 #include <atomic> 20 #include "plugin/plugin_time.h" 21 #include "avcodec_common.h" 22 #include "surface/surface.h" 23 #include "osal/task/condition_variable.h" 24 #include "osal/task/mutex.h" 25 #include "osal/task/task.h" 26 #include "video_sink.h" 27 #include "sink/media_synchronous_sink.h" 28 #include "common/status.h" 29 #include "video_decoder_adapter.h" 30 #include "meta/meta.h" 31 #include "meta/format.h" 32 #include "filter/filter.h" 33 #include "media_sync_manager.h" 34 #include "common/media_core.h" 35 #include "common/seek_callback.h" 36 #include "drm_i_keysession_service.h" 37 #include "interrupt_listener.h" 38 #include "sei_parser_helper.h" 39 #include "post_processor/base_video_post_processor.h" 40 #include "post_processor/video_post_processor_factory.h" 41 42 namespace OHOS { 43 namespace Media { 44 namespace Pipeline { 45 class DecoderSurfaceFilter : public Filter, public std::enable_shared_from_this<DecoderSurfaceFilter>, 46 public InterruptListener { 47 public: 48 explicit DecoderSurfaceFilter(const std::string& name, FilterType type); 49 ~DecoderSurfaceFilter() override; 50 51 void Init(const std::shared_ptr<EventReceiver> &receiver, 52 const std::shared_ptr<FilterCallback> &callback) override; 53 Status Configure(const std::shared_ptr<Meta> ¶meter); 54 Status DoInitAfterLink() override; 55 Status DoPrepare() override; 56 Status DoStart() override; 57 Status DoPause() override; 58 Status DoPauseDragging() override; 59 Status DoResume() override; 60 Status DoResumeDragging() override; 61 Status DoStop() override; 62 Status DoFlush() override; 63 Status DoRelease() override; 64 Status DoPreroll() override; 65 Status DoWaitPrerollDone(bool render) override; 66 Status DoSetPlayRange(int64_t start, int64_t end) override; 67 Status DoProcessInputBuffer(int recvArg, bool dropFrame) override; 68 Status DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) override; 69 Status DoSetPerfRecEnabled(bool isPerfRecEnabled) override; 70 71 void SetParameter(const std::shared_ptr<Meta>& parameter) override; 72 void GetParameter(std::shared_ptr<Meta>& parameter) override; 73 void OnInterrupted(bool isInterruptNeeded) override; 74 75 Status LinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 76 Status UpdateNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 77 Status UnLinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 78 void OnLinkedResult(const sptr<AVBufferQueueProducer> &outputBufferQueue, std::shared_ptr<Meta> &meta); 79 void OnUpdatedResult(std::shared_ptr<Meta> &meta); 80 81 void OnUnlinkedResult(std::shared_ptr<Meta> &meta); 82 FilterType GetFilterType(); 83 void DrainOutputBuffer(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 84 void DecoderDrainOutputBuffer(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 85 Status SetVideoSurface(sptr<Surface> videoSurface); 86 87 Status SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySessionProxy, 88 bool svp); 89 90 void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode); 91 void PostProcessorOnError(int32_t errorCode); 92 93 sptr<AVBufferQueueProducer> GetInputBufferQueue(); 94 void SetSyncCenter(std::shared_ptr<MediaSyncManager> syncCenter); 95 void SetSeekTime(int64_t seekTimeUs); 96 void ResetSeekInfo(); 97 Status HandleInputBuffer(); 98 void OnDumpInfo(int32_t fd); 99 100 void SetCallingInfo(int32_t appUid, int32_t appPid, std::string bundleName, uint64_t instanceId); 101 102 Status GetLagInfo(int32_t& lagTimes, int32_t& maxLagDuration, int32_t& avgLagDuration); 103 void SetBitrateStart(); 104 void OnOutputFormatChanged(const MediaAVCodec::Format &format); 105 Status StartSeekContinous(); 106 Status StopSeekContinous(); 107 void RegisterVideoFrameReadyCallback(std::shared_ptr<VideoFrameReadyCallback> &callback); 108 void DeregisterVideoFrameReadyCallback(); 109 int32_t GetDecRateUpperLimit(); 110 bool GetIsSupportSeekWithoutFlush(); 111 void ConsumeVideoFrame(uint32_t index, bool isRender, int64_t renderTimeNs = 0L); 112 Status SetSeiMessageCbStatus(bool status, const std::vector<int32_t> &payloadTypes); 113 114 Status InitPostProcessor(); 115 void SetPostProcessorType(VideoPostProcessorType type); 116 Status SetPostProcessorOn(bool isSuperResolutionOn); 117 Status SetVideoWindowSize(int32_t width, int32_t height); 118 119 protected: 120 Status OnLinked(StreamType inType, const std::shared_ptr<Meta> &meta, 121 const std::shared_ptr<FilterLinkCallback> &callback) override; 122 Status OnUpdated(StreamType inType, const std::shared_ptr<Meta> &meta, 123 const std::shared_ptr<FilterLinkCallback> &callback) override; 124 Status OnUnLinked(StreamType inType, const std::shared_ptr<FilterLinkCallback>& callback) override; 125 126 private: 127 void RenderLoop(); 128 std::string GetCodecName(std::string mimeType); 129 int64_t CalculateNextRender(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 130 void ParseDecodeRateLimit(); 131 void RenderNextOutput(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 132 Status ReleaseOutputBuffer(int index, bool render, const std::shared_ptr<AVBuffer> &outBuffer, int64_t renderTime); 133 void DoReleaseOutputBuffer(uint32_t index, bool render); 134 void DoRenderOutputBufferAtTime(uint32_t index, int64_t renderTime); 135 bool AcquireNextRenderBuffer(bool byIdx, uint32_t &index, std::shared_ptr<AVBuffer> &outBuffer, 136 int64_t renderTime = 0); 137 bool DrainSeekContinuous(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 138 bool DrainPreroll(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 139 bool DrainSeekClosest(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 140 void HandleFirstOutput(); 141 void HandleEosOutput(int index); 142 void ReportEosEvent(); 143 void RenderAtTimeDfx(int64_t renderTimeNs, int64_t currentTimeNs, int64_t lastRenderTimeNs); 144 int64_t GetSystimeTimeNs(); 145 bool IsPostProcessorSupported(); 146 std::shared_ptr<BaseVideoPostProcessor> CreatePostProcessor(); 147 148 std::string name_; 149 FilterType filterType_; 150 std::shared_ptr<EventReceiver> eventReceiver_; 151 std::shared_ptr<FilterCallback> filterCallback_; 152 std::shared_ptr<FilterLinkCallback> onLinkedResultCallback_; 153 std::shared_ptr<VideoDecoderAdapter> videoDecoder_; 154 std::shared_ptr<VideoSink> videoSink_; 155 std::string codecMimeType_; 156 std::shared_ptr<Meta> configureParameter_; 157 std::shared_ptr<Meta> meta_; 158 159 std::shared_ptr<Filter> nextFilter_; 160 161 Format configFormat_; 162 163 std::atomic<uint64_t> renderFrameCnt_{0}; 164 std::atomic<uint64_t> discardFrameCnt_{0}; 165 std::atomic<bool> isSeek_{false}; 166 int64_t seekTimeUs_{0}; 167 168 bool refreshTotalPauseTime_{false}; 169 int64_t latestBufferTime_{HST_TIME_NONE}; 170 int64_t latestPausedTime_{HST_TIME_NONE}; 171 int64_t totalPausedTime_{0}; 172 int64_t stopTime_{0}; 173 sptr<Surface> decoderOutputSurface_; 174 sptr<Surface> videoSurface_; 175 bool isDrmProtected_ = false; 176 #ifdef SUPPORT_DRM 177 sptr<DrmStandard::IMediaKeySessionService> keySessionServiceProxy_ = nullptr; 178 #endif 179 bool svpFlag_ = false; 180 std::atomic<bool> isPaused_{false}; 181 std::list<std::pair<int, std::shared_ptr<AVBuffer>>> outputBuffers_; 182 std::mutex mutex_; 183 std::unique_ptr<std::thread> readThread_ = nullptr; 184 std::atomic<bool> isThreadExit_ = true; 185 std::condition_variable condBufferAvailable_; 186 187 std::atomic<bool> isRenderStarted_{false}; 188 std::atomic<bool> isInterruptNeeded_{false}; 189 Mutex formatChangeMutex_{}; 190 int32_t rateUpperLimit_{0}; 191 192 std::mutex prerollMutex_ {}; 193 std::atomic<bool> inPreroll_ {false}; 194 std::condition_variable prerollDoneCond_ {}; 195 std::atomic<bool> prerollDone_ {true}; 196 std::atomic<bool> eosNext_ {false}; 197 bool isFirstFrameAfterResume_ {true}; 198 199 int32_t appUid_ = -1; 200 int32_t appPid_ = -1; 201 std::string bundleName_; 202 uint64_t instanceId_ = 0; 203 int64_t playRangeStartTime_ = -1; 204 int64_t playRangeEndTime_ = -1; 205 206 std::atomic<int32_t> bitrateChange_{0}; 207 int32_t surfaceWidth_{0}; 208 int32_t surfaceHeight_{0}; 209 210 std::shared_ptr<VideoFrameReadyCallback> videoFrameReadyCallback_; 211 bool isInSeekContinous_{false}; 212 std::unordered_map<uint32_t, std::shared_ptr<AVBuffer>> outputBufferMap_; 213 std::mutex draggingMutex_ {}; 214 std::unique_ptr<Task> eosTask_ {nullptr}; 215 std::atomic<int64_t> lastRenderTimeNs_ = HST_TIME_NONE; 216 int64_t renderTimeMaxAdvanceUs_ { 80000 }; 217 bool enableRenderAtTime_ {true}; 218 bool enableRenderAtTimeDfx_ {false}; 219 std::list<int64_t> renderTimeQueue_; 220 std::string logMessage; 221 bool seiMessageCbStatus_ {false}; 222 std::vector<int32_t> payloadTypes_ {}; 223 sptr<SeiParserListener> producerListener_ {}; 224 sptr<Media::AVBufferQueueProducer> inputBufferQueueProducer_ {}; 225 226 static constexpr int32_t DEFAULT_TARGET_WIDTH = 1920; 227 static constexpr int32_t DEFAULT_TARGET_HEIGHT = 1080; 228 int32_t postProcessorTargetWidth_ {DEFAULT_TARGET_WIDTH}; 229 int32_t postProcessorTargetHeight_ {DEFAULT_TARGET_HEIGHT}; 230 bool isPostProcessorOn_ {false}; 231 bool isPostProcessorSupported_ {true}; 232 VideoPostProcessorType postProcessorType_ { VideoPostProcessorType::NONE }; 233 std::shared_ptr<BaseVideoPostProcessor> postProcessor_; 234 }; 235 } // namespace Pipeline 236 } // namespace Media 237 } // namespace OHOS 238 #endif // MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 239