1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 17 #define MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 18 19 #include <atomic> 20 #include "plugin/plugin_time.h" 21 #include "avcodec_common.h" 22 #include "surface/surface.h" 23 #include "osal/task/condition_variable.h" 24 #include "osal/task/mutex.h" 25 #include "osal/task/task.h" 26 #include "video_sink.h" 27 #include "sink/media_synchronous_sink.h" 28 #include "common/status.h" 29 #include "video_decoder_adapter.h" 30 #include "meta/meta.h" 31 #include "meta/format.h" 32 #include "filter/filter.h" 33 #include "media_sync_manager.h" 34 #include "common/media_core.h" 35 #include "common/seek_callback.h" 36 #include "drm_i_keysession_service.h" 37 #include "interrupt_listener.h" 38 #include "sei_parser_helper.h" 39 #include "post_processor/base_video_post_processor.h" 40 #include "post_processor/video_post_processor_factory.h" 41 #include "common/fdsan_fd.h" 42 43 namespace OHOS { 44 namespace Media { 45 namespace Pipeline { 46 class DecoderSurfaceFilter : public Filter, public std::enable_shared_from_this<DecoderSurfaceFilter>, 47 public InterruptListener { 48 public: 49 explicit DecoderSurfaceFilter(const std::string& name, FilterType type); 50 ~DecoderSurfaceFilter() override; 51 52 void Init(const std::shared_ptr<EventReceiver> &receiver, 53 const std::shared_ptr<FilterCallback> &callback) override; 54 Status Configure(const std::shared_ptr<Meta> ¶meter); 55 Status DoInitAfterLink() override; 56 Status DoPrepare() override; 57 Status DoStart() override; 58 Status DoPause() override; 59 Status DoFreeze() override; 60 Status DoUnFreeze() override; 61 Status DoPauseDragging() override; 62 Status DoResume() override; 63 Status DoResumeDragging() override; 64 Status DoStop() override; 65 Status DoFlush() override; 66 Status DoRelease() override; 67 Status DoPreroll() override; 68 Status DoWaitPrerollDone(bool render) override; 69 Status DoSetPlayRange(int64_t start, int64_t end) override; 70 Status DoProcessInputBuffer(int recvArg, bool dropFrame) override; 71 Status DoProcessOutputBuffer(int recvArg, bool dropFrame, bool byIdx, uint32_t idx, int64_t renderTime) override; 72 Status DoSetPerfRecEnabled(bool isPerfRecEnabled) override; 73 74 void SetParameter(const std::shared_ptr<Meta>& parameter) override; 75 void GetParameter(std::shared_ptr<Meta>& parameter) override; 76 void OnInterrupted(bool isInterruptNeeded) override; 77 78 Status LinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 79 Status UpdateNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 80 Status UnLinkNext(const std::shared_ptr<Filter> &nextFilter, StreamType outType) override; 81 void OnLinkedResult(const sptr<AVBufferQueueProducer> &outputBufferQueue, std::shared_ptr<Meta> &meta); 82 void OnUpdatedResult(std::shared_ptr<Meta> &meta); 83 84 void OnUnlinkedResult(std::shared_ptr<Meta> &meta); 85 FilterType GetFilterType(); 86 void DrainOutputBuffer(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 87 void DecoderDrainOutputBuffer(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 88 Status SetVideoSurface(sptr<Surface> videoSurface); 89 90 Status SetDecryptConfig(const sptr<DrmStandard::IMediaKeySessionService> &keySessionProxy, 91 bool svp); 92 93 void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode); 94 void PostProcessorOnError(int32_t errorCode); 95 96 sptr<AVBufferQueueProducer> GetInputBufferQueue(); 97 void SetSyncCenter(std::shared_ptr<MediaSyncManager> syncCenter); 98 void SetSeekTime(int64_t seekTimeUs, PlayerSeekMode mode = PlayerSeekMode::SEEK_CLOSEST); 99 void ResetSeekInfo(); 100 Status HandleInputBuffer(); 101 void OnDumpInfo(int32_t fd); 102 103 void SetCallingInfo(int32_t appUid, int32_t appPid, std::string bundleName, uint64_t instanceId); 104 105 Status GetLagInfo(int32_t& lagTimes, int32_t& maxLagDuration, int32_t& avgLagDuration); 106 void SetBitrateStart(); 107 void OnOutputFormatChanged(const MediaAVCodec::Format &format); 108 Status StartSeekContinous(); 109 Status StopSeekContinous(); 110 void RegisterVideoFrameReadyCallback(std::shared_ptr<VideoFrameReadyCallback> &callback); 111 void DeregisterVideoFrameReadyCallback(); 112 int32_t GetDecRateUpperLimit(); 113 bool GetIsSupportSeekWithoutFlush(); 114 void ConsumeVideoFrame(uint32_t index, bool isRender, int64_t renderTimeNs = 0L); 115 Status SetSeiMessageCbStatus(bool status, const std::vector<int32_t> &payloadTypes); 116 117 Status InitPostProcessor(); 118 void SetPostProcessorType(VideoPostProcessorType type); 119 Status SetPostProcessorOn(bool isSuperResolutionOn); 120 Status SetVideoWindowSize(int32_t width, int32_t height); 121 void NotifyAudioComplete(); 122 Status SetSpeed(float speed); 123 Status SetPostProcessorFd(int32_t postProcessorFd); 124 Status SetCameraPostprocessing(bool enable); 125 Status SetCameraPostprocessingDirect(bool enable); 126 void NotifyPause(); 127 void NotifyMemoryExchange(bool exchangeFlag); 128 Status SetMediaMuted(bool isMuted, bool hasInitialized); 129 Status DoReleaseOnMuted(bool isNeedRelease) override; 130 Status DoReInitAndStart() override; 131 132 protected: 133 Status OnLinked(StreamType inType, const std::shared_ptr<Meta> &meta, 134 const std::shared_ptr<FilterLinkCallback> &callback) override; 135 Status OnUpdated(StreamType inType, const std::shared_ptr<Meta> &meta, 136 const std::shared_ptr<FilterLinkCallback> &callback) override; 137 Status OnUnLinked(StreamType inType, const std::shared_ptr<FilterLinkCallback>& callback) override; 138 139 private: 140 void RenderLoop(); 141 std::string GetCodecName(std::string mimeType); 142 int64_t CalculateNextRender(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 143 void ParseDecodeRateLimit(); 144 void RenderNextOutput(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 145 Status ReleaseOutputBuffer(int index, bool render, const std::shared_ptr<AVBuffer> &outBuffer, int64_t renderTime); 146 void DoReleaseOutputBuffer(uint32_t index, bool render, int64_t pts = 0); 147 void DoRenderOutputBufferAtTime(uint32_t index, int64_t renderTime, int64_t pts = 0); 148 bool AcquireNextRenderBuffer(bool byIdx, uint32_t &index, std::shared_ptr<AVBuffer> &outBuffer, 149 int64_t renderTime = 0); 150 bool DrainSeekContinuous(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 151 bool DrainPreroll(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 152 bool DrainSeekClosest(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 153 void HandleFirstOutput(); 154 void HandleEosOutput(int index); 155 void ReportEosEvent(); 156 void RenderAtTimeDfx(int64_t renderTimeNs, int64_t currentTimeNs, int64_t lastRenderTimeNs); 157 int64_t GetSystimeTimeNs(); 158 bool IsPostProcessorSupported(); 159 std::shared_ptr<BaseVideoPostProcessor> CreatePostProcessor(); 160 void InitPostProcessorType(); 161 #ifdef SUPPORT_CAMERA_POST_PROCESSOR 162 void LoadCameraPostProcessorLib(); 163 #endif 164 Status CheckBufferDecodedCorrectly(uint32_t index, std::shared_ptr<AVBuffer> &outputBuffer); 165 166 std::string name_; 167 FilterType filterType_; 168 std::shared_ptr<EventReceiver> eventReceiver_; 169 std::shared_ptr<FilterCallback> filterCallback_; 170 std::shared_ptr<FilterLinkCallback> onLinkedResultCallback_; 171 std::shared_ptr<VideoDecoderAdapter> videoDecoder_; 172 std::shared_ptr<VideoSink> videoSink_; 173 std::string codecMimeType_; 174 std::shared_ptr<Meta> configureParameter_; 175 std::shared_ptr<Meta> meta_; 176 177 std::shared_ptr<Filter> nextFilter_; 178 179 Format configFormat_; 180 181 std::atomic<uint64_t> renderFrameCnt_{0}; 182 std::atomic<uint64_t> discardFrameCnt_{0}; 183 std::atomic<bool> isSeek_{false}; 184 int64_t seekTimeUs_{0}; 185 186 bool refreshTotalPauseTime_{false}; 187 int64_t latestBufferTime_{HST_TIME_NONE}; 188 int64_t latestPausedTime_{HST_TIME_NONE}; 189 int64_t totalPausedTime_{0}; 190 int64_t stopTime_{0}; 191 sptr<Surface> decoderOutputSurface_; 192 sptr<Surface> videoSurface_; 193 bool isDrmProtected_ = false; 194 #ifdef SUPPORT_DRM 195 sptr<DrmStandard::IMediaKeySessionService> keySessionServiceProxy_ = nullptr; 196 #endif 197 bool svpFlag_ = false; 198 std::atomic<bool> isPaused_{false}; 199 std::list<std::pair<int, std::shared_ptr<AVBuffer>>> outputBuffers_; 200 std::mutex mutex_; 201 std::unique_ptr<std::thread> readThread_ = nullptr; 202 std::atomic<bool> isThreadExit_ = true; 203 std::condition_variable condBufferAvailable_; 204 205 std::atomic<bool> isRenderStarted_{false}; 206 std::atomic<bool> isInterruptNeeded_{false}; 207 Mutex formatChangeMutex_{}; 208 int32_t rateUpperLimit_{0}; 209 210 std::mutex prerollMutex_ {}; 211 std::atomic<bool> inPreroll_ {false}; 212 std::condition_variable prerollDoneCond_ {}; 213 std::atomic<bool> prerollDone_ {true}; 214 std::atomic<bool> eosNext_ {false}; 215 bool isFirstFrameAfterResume_ {true}; 216 217 int32_t appUid_ = -1; 218 int32_t appPid_ = -1; 219 std::string bundleName_; 220 uint64_t instanceId_ = 0; 221 int64_t playRangeStartTime_ = -1; 222 int64_t playRangeEndTime_ = -1; 223 224 std::atomic<int32_t> bitrateChange_{0}; 225 int32_t surfaceWidth_{0}; 226 int32_t surfaceHeight_{0}; 227 228 std::shared_ptr<VideoFrameReadyCallback> videoFrameReadyCallback_; 229 bool isInSeekContinous_{false}; 230 std::unordered_map<uint32_t, std::shared_ptr<AVBuffer>> outputBufferMap_; 231 std::mutex draggingMutex_ {}; 232 std::unique_ptr<Task> eosTask_ {nullptr}; 233 std::atomic<int64_t> lastRenderTimeNs_ = HST_TIME_NONE; 234 int64_t renderTimeMaxAdvanceUs_ { 80000 }; 235 bool enableRenderAtTime_ {true}; 236 bool enableRenderAtTimeDfx_ {false}; 237 std::list<int64_t> renderTimeQueue_; 238 std::string logMessage; 239 bool seiMessageCbStatus_ {false}; 240 std::vector<int32_t> payloadTypes_ {}; 241 sptr<SeiParserListener> producerListener_ {}; 242 sptr<Media::AVBufferQueueProducer> inputBufferQueueProducer_ {}; 243 244 static constexpr int32_t DEFAULT_TARGET_WIDTH = 1920; 245 static constexpr int32_t DEFAULT_TARGET_HEIGHT = 1080; 246 int32_t postProcessorTargetWidth_ {DEFAULT_TARGET_WIDTH}; 247 int32_t postProcessorTargetHeight_ {DEFAULT_TARGET_HEIGHT}; 248 bool isPostProcessorOn_ {false}; 249 bool isPostProcessorSupported_ {true}; 250 VideoPostProcessorType postProcessorType_ { VideoPostProcessorType::NONE }; 251 std::shared_ptr<BaseVideoPostProcessor> postProcessor_; 252 std::atomic<bool> enableCameraPostprocessing_ {false}; 253 bool isCameraPostProcessorSupported_ {true}; 254 255 int64_t eosPts_ {INT64_MAX}; 256 int64_t prevDecoderPts_ {INT64_MAX}; 257 std::mutex fdMutex_ {}; 258 std::unique_ptr<FdsanFd> fdsanFd_ = nullptr; 259 int32_t preScaleType_ {0}; 260 #ifdef SUPPORT_CAMERA_POST_PROCESSOR 261 std::mutex loadLibMutex_ {}; 262 static void *cameraPostProcessorLibHandle_; 263 #endif 264 265 std::atomic<bool> isVideoMuted_ {false}; 266 bool isDecoderReleasedForMute_ {true}; 267 bool hasReceivedReleaseEvent_ {false}; 268 bool isFirstStart_ = true; 269 }; 270 } // namespace Pipeline 271 } // namespace Media 272 } // namespace OHOS 273 #endif // MEDIA_PIPELINE_VIDEO_SINK_FILTER_H 274