1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef MEDIA_DEMUXER_H 17 #define MEDIA_DEMUXER_H 18 19 #include <atomic> 20 #include <limits> 21 #include <string> 22 #include <shared_mutex> 23 #include <unordered_set> 24 25 #include "osal/task/condition_variable.h" 26 #include "avcodec_common.h" 27 #include "buffer/avbuffer.h" 28 #include "common/media_source.h" 29 #include "common/seek_callback.h" 30 #include "demuxer/type_finder.h" 31 #include "demuxer/sample_queue.h" 32 #include "filter/filter.h" 33 #include "meta/media_types.h" 34 #include "osal/task/autolock.h" 35 #include "osal/task/task.h" 36 #include "plugin/plugin_base.h" 37 #include "plugin/plugin_info.h" 38 #include "plugin/plugin_time.h" 39 #include "plugin/demuxer_plugin.h" 40 #include "interrupt_listener.h" 41 #include "performance_utils.h" 42 #include "media_sync_manager.h" 43 44 namespace OHOS { 45 namespace Media { 46 using MediaSource = OHOS::Media::Plugins::MediaSource; 47 using MediaSyncManager = OHOS::Media::Pipeline::MediaSyncManager; 48 using FileType = OHOS::Media::Plugins::FileType; 49 class BaseStreamDemuxer; 50 class DemuxerPluginManager; 51 class Source; 52 53 class AVBufferQueueProducer; 54 enum class DemuxerTrackType : uint32_t { 55 VIDEO = 0, 56 AUDIO, 57 SUBTITLE, 58 }; 59 60 class MediaDemuxer : public std::enable_shared_from_this<MediaDemuxer>, 61 public Plugins::Callback, 62 public InterruptListener, 63 public SampleQueueCallback { 64 public: 65 explicit MediaDemuxer(); 66 ~MediaDemuxer() override; 67 68 Status SetDataSource(const std::shared_ptr<MediaSource> &source); 69 Status SetSubtitleSource(const std::shared_ptr<MediaSource> &source); 70 void SetBundleName(const std::string& bundleName); 71 Status SetOutputBufferQueue(int32_t trackId, const sptr<AVBufferQueueProducer>& producer); 72 73 std::shared_ptr<Meta> GetGlobalMetaInfo(); 74 std::vector<std::shared_ptr<Meta>> GetStreamMetaInfo() const; 75 std::shared_ptr<Meta> GetUserMeta(); 76 77 Status SeekTo(int64_t seekTime, Plugins::SeekMode mode, int64_t& realSeekTime); 78 Status Reset(); 79 Status Start(); 80 Status Stop(); 81 Status Pause(); 82 Status PauseDragging(); 83 Status PauseAudioAlign(); 84 Status Resume(); 85 Status ResumeDragging(); 86 Status ResumeAudioAlign(); 87 Status Flush(); 88 Status Preroll(); 89 Status PausePreroll(); 90 91 Status StartTask(int32_t trackId); 92 Status SelectTrack(int32_t trackId); 93 Status UnselectTrack(int32_t trackId); 94 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample); 95 Status GetBitRates(std::vector<uint32_t> &bitRates); 96 Status SelectBitRate(uint32_t bitRate); 97 Status StopBufferring(bool flag); 98 Status GetDownloadInfo(DownloadInfo& downloadInfo); 99 Status GetPlaybackInfo(PlaybackInfo& playbackInfo); 100 Status GetMediaKeySystemInfo(std::multimap<std::string, std::vector<uint8_t>> &infos); 101 void SetDrmCallback(const std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> &callback); 102 void HandleEvent(const Plugins::PluginEvent &event); 103 void OnEvent(const Plugins::PluginEvent &event) override; 104 void OnEventBuffer(const Plugins::PluginEvent &event, std::shared_ptr<Pipeline::EventReceiver> eventReceiver); 105 void OnSeekReadyEvent(const Plugins::PluginEvent &event); 106 void OnDashSeekReadyEvent(const Plugins::PluginEvent &event); 107 void OnHlsSeekReadyEvent(const Plugins::PluginEvent &event); 108 void OnDfxEvent(const Plugins::PluginDfxEvent &event) override; 109 110 Status SetPerfRecEnabled(bool isPerfRecEnabled); 111 112 std::map<uint32_t, sptr<AVBufferQueueProducer>> GetBufferQueueProducerMap(); 113 Status PauseTaskByTrackId(int32_t trackId); 114 bool IsRenderNextVideoFrameSupported(); 115 116 void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver); 117 bool GetDuration(int64_t& durationMs); 118 void SetPlayerId(std::string playerId); 119 void SetDumpInfo(bool isDump, uint64_t instanceId); 120 121 Status OptimizeDecodeSlow(bool isDecodeOptimizationEnabled); 122 Status SetDecoderFramerateUpperLimit(int32_t decoderFramerateUpperLimit, uint32_t trackId); 123 Status SetSpeed(float speed); 124 Status SetFrameRate(double framerate, uint32_t trackId); 125 void OnInterrupted(bool isInterruptNeeded) override; 126 void OnDumpInfo(int32_t fd); 127 bool IsLocalDrmInfosExisted(); 128 Status DisableMediaTrack(Plugins::MediaType mediaType); 129 void OnBufferAvailable(uint32_t trackId); 130 131 void SetSelectBitRateFlag(bool flag, uint32_t desBitRate) override; 132 bool CanAutoSelectBitRate() override; 133 134 bool IsRefParserSupported(); 135 Status StartReferenceParser(int64_t startTimeMs, bool isForward = true); 136 Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo); 137 Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo); 138 Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo); 139 bool IsVideoEos(); 140 bool HasEosTrack(); 141 Status GetIFramePos(std::vector<uint32_t> &IFramePos); 142 Status Dts2FrameId(int64_t dts, uint32_t &frameId); 143 Status SeekMs2FrameId(int64_t seekMs, uint32_t &frameId); 144 Status FrameId2SeekMs(uint32_t frameId, int64_t &seekMs); 145 void RegisterVideoStreamReadyCallback(const std::shared_ptr<VideoStreamReadyCallback> &callback); 146 void DeregisterVideoStreamReadyCallback(); 147 148 Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex, 149 const uint64_t relativePresentationTimeUs, uint32_t &index); 150 Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex, 151 const uint32_t index, uint64_t &relativePresentationTimeUs); 152 Status ResumeDemuxerReadLoop(); 153 Status PauseDemuxerReadLoop(); 154 void SetCacheLimit(uint32_t limitSize); 155 void SetEnableOnlineFdCache(bool isEnableFdCache); 156 void WaitForBufferingEnd(); 157 int32_t GetCurrentVideoTrackId(); 158 uint32_t GetTargetVideoTrackId(std::vector<std::shared_ptr<Meta>> trackInfos); 159 void SetIsEnableReselectVideoTrack(bool isEnable); 160 void SetApiVersion(int32_t apiVersion); 161 bool IsLocalFd(); 162 163 void SetSyncCenter(std::shared_ptr<MediaSyncManager> syncCenter); 164 bool IsFlvLiveStream(); 165 Status RebootPlugin(); 166 uint64_t GetCachedDuration(); 167 void RestartAndClearBuffer(); 168 bool IsFlvLive(); 169 private: 170 class AVBufferQueueProducerListener; 171 class TrackWrapper; 172 struct MediaMetaData { 173 std::vector<std::shared_ptr<Meta>> trackMetas; 174 std::shared_ptr<Meta> globalMeta; 175 }; 176 177 struct MaintainBaseInfo { 178 int64_t segmentOffset = -1; 179 int64_t basePts = -1; 180 int64_t candidateBasePts = -1; 181 int64_t lastPts = 0; 182 int64_t lastPtsModifyedMax = -1; 183 bool isLastPtsChange = false; 184 }; 185 186 struct SyncFrameInfo { 187 int64_t pts = -1; 188 int32_t skipOpenGopUnrefFrameCnt = 0; 189 }; 190 191 bool isHttpSource_ = false; 192 std::string videoMime_{}; 193 194 static constexpr uint32_t TRACK_ID_DUMMY = std::numeric_limits<uint32_t>::max(); 195 static constexpr int32_t DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT = 120; 196 197 Status InnerPrepare(); 198 void InitMediaMetaData(const Plugins::MediaInfo& mediaInfo); 199 void InitDefaultTrack(const Plugins::MediaInfo& mediaInfo, uint32_t& videoTrackId, 200 uint32_t& audioTrackId, uint32_t& subtitleTrackId, std::string& videoMime); 201 bool IsOffsetValid(int64_t offset) const; 202 std::shared_ptr<Meta> GetTrackMeta(uint32_t trackId); 203 Status AddDemuxerCopyTask(uint32_t trackId, TaskType type); 204 Status AddDemuxerCopyTaskByTrack(uint32_t trackId, DemuxerTrackType type); 205 206 Status StopAllTask(); 207 Status PauseAllTask(); 208 Status PauseAllTaskAsync(); 209 Status ResumeAllTask(); 210 void AccelerateTrackTask(uint32_t trackId); 211 void SetTrackNotifyFlag(uint32_t trackId, bool isNotifyNeeded); 212 void AccelerateSampleConsumerTask(uint32_t trackId); 213 void ResetInner(); 214 215 bool GetDrmInfosUpdated(const std::multimap<std::string, std::vector<uint8_t>> &newInfos, 216 std::multimap<std::string, std::vector<uint8_t>> &result); 217 Status ProcessDrmInfos(); 218 void HandleSourceDrmInfoEvent(const std::multimap<std::string, std::vector<uint8_t>> &info); 219 Status ReportDrmInfos(const std::multimap<std::string, std::vector<uint8_t>> &info); 220 221 bool HasVideo(); 222 void DumpBufferToFile(uint32_t trackId, std::shared_ptr<AVBuffer> buffer); 223 bool IsBufferDroppable(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 224 bool CheckDropAudioFrame(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 225 bool IsTrackDisabled(Plugins::MediaType mediaType); 226 bool CheckTrackEnabledById(uint32_t trackId); 227 bool HandleDashChangeStream(uint32_t trackId); 228 229 Status SeekToTimeAfter(); 230 bool SelectBitRateChangeStream(uint32_t trackId); 231 bool SelectTrackChangeStream(uint32_t trackId); 232 bool HandleSelectTrackChangeStream(int32_t trackId, int32_t newStreamID, int32_t& newTrackId); 233 void HandleSelectTrackStreamSeek(int32_t streamID, int32_t& trackId); 234 std::shared_ptr<Plugins::DemuxerPlugin> GetCurFFmpegPlugin(); 235 bool IsNeedMapToInnerTrackID(); 236 int64_t GetReadLoopRetryUs(uint32_t trackId); 237 uint64_t GetSampleQueueDuration(); 238 void UpdateSampleQueueCache(); 239 240 Plugins::Seekable seekable_; 241 Plugins::Seekable subSeekable_; 242 std::string uri_; 243 std::string SubtitleUri_; 244 uint64_t mediaDataSize_; 245 uint64_t subMediaDataSize_; 246 247 std::shared_ptr<MediaSource> mediaSource_; 248 std::shared_ptr<Source> source_; 249 std::shared_ptr<Source> subtitleSource_; 250 MediaMetaData mediaMetaData_; 251 252 int64_t ReadLoop(uint32_t trackId); 253 Status CopyFrameToUserQueue(uint32_t trackId); 254 bool GetBufferFromUserQueue(uint32_t queueIndex, uint32_t size = 0); 255 Status InnerReadSample(uint32_t trackId, std::shared_ptr<AVBuffer>); 256 Status InnerSelectTrack(int32_t trackId); 257 Status HandleRead(uint32_t trackId); 258 int64_t ParserRefInfo(); 259 void TryReclaimParserTask(); 260 261 Status HandleSelectTrack(int32_t trackId); 262 Status HandleDashSelectTrack(int32_t trackId); 263 Status DoSelectTrack(int32_t trackId, int32_t curTrackId); 264 Status HandleRebootPlugin(int32_t trackId, bool& isRebooted); 265 Status HandleHlsRebootPlugin(); 266 Status HandleSeekChangeStream(int32_t currentStreamId, int32_t newStreamId, int32_t trackId); 267 268 bool IsSubtitleMime(const std::string& mime); 269 void HandleAutoMaintainPts(uint32_t trackeId, std::shared_ptr<AVBuffer> sample); 270 void InitPtsInfo(); 271 void InitMediaStartPts(); 272 void UpdateBufferQueueListener(int32_t trackId); 273 bool IsOpenGopBufferDroppable(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 274 void UpdateSyncFrameInfo(std::shared_ptr<AVBuffer> sample, uint32_t trackId, bool isDiscardable = false); 275 void EnterDraggingOpenGopCnt(); 276 void ResetDraggingOpenGopCnt(); 277 Status ReadSampleWithPerfRecord(const std::shared_ptr<Plugins::DemuxerPlugin> &pluginTemp, 278 const int32_t &innerTrackID, const std::shared_ptr<AVBuffer> &sample); 279 Status HandleTrackEos(uint32_t trackId); 280 void SetOutputBufferPts(std::shared_ptr<AVBuffer> &outputBuffer); 281 282 Status AddSampleBufferQueue(uint32_t trackId); 283 int64_t SampleConsumerLoop(uint32_t trackId); 284 void SetTrackNotifySampleConsumerFlag(uint32_t trackId, bool isNotifySampleConsumerNeeded); 285 bool IsRightMediaTrack(uint32_t trackId, DemuxerTrackType type) const; 286 int64_t GetLastVideoBufferAbsPts(uint32_t trackId) const; 287 void UpdateLastVideoBufferAbsPts(uint32_t trackId); 288 Status OnSelectBitrateOk(int64_t startPts, uint32_t bitRate) override; 289 Status OnSampleQueueBufferAvailable(uint32_t queueId) override; 290 Status OnSampleQueueBufferConsume(uint32_t queueId) override; 291 Status HandleSelectBitrateBySampleQueue(int64_t startPts, uint32_t bitrate); 292 bool IsIgonreBuffering(); 293 uint64_t demuxerCacheDuration_ = 0; 294 uint64_t sourceCacheDuration_ = 0; 295 int64_t lastClockTimeMs_ = 0; 296 297 Mutex mapMutex_{}; 298 std::map<uint32_t, std::shared_ptr<TrackWrapper>> trackMap_; 299 std::map<uint32_t, sptr<AVBufferQueueProducer>> bufferQueueMap_; 300 std::map<uint32_t, std::shared_ptr<AVBuffer>> bufferMap_; 301 302 std::map<uint32_t, std::shared_ptr<SampleQueue>> sampleQueueMap_; 303 std::map<uint32_t, bool> eosMap_; 304 std::map<uint32_t, uint32_t> requestBufferErrorCountMap_; 305 std::atomic<bool> isThreadExit_ = true; 306 bool useBufferQueue_ = false; 307 bool isAccurateSeekForHLS_ = false; 308 int64_t videoStartTime_{0}; 309 310 std::shared_mutex drmMutex{}; 311 std::mutex isSelectTrackMutex_{}; 312 std::mutex rebootPluginMutex_{}; 313 std::multimap<std::string, std::vector<uint8_t>> localDrmInfos_; 314 std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> drmCallback_; 315 316 std::map<uint32_t, std::unique_ptr<Task>> taskMap_; 317 std::map<uint32_t, std::unique_ptr<Task>> sampleConsumerTaskMap_; 318 std::shared_ptr<MediaSyncManager> syncCenter_; 319 bool isFlvLiveStream_ = false; 320 std::unique_ptr<Task> notifyBitrateTask_; 321 322 std::shared_ptr<Pipeline::EventReceiver> eventReceiver_; 323 int64_t lastSeekTime_{Plugins::HST_TIME_NONE}; 324 bool isSeeked_{false}; 325 uint32_t videoTrackId_{TRACK_ID_DUMMY}; 326 uint32_t audioTrackId_{TRACK_ID_DUMMY}; 327 uint32_t subtitleTrackId_{TRACK_ID_DUMMY}; 328 bool firstAudio_{true}; 329 std::mutex stopMutex_ {}; 330 331 std::atomic<bool> isStopped_ = true; 332 std::atomic<bool> isPaused_ = false; 333 std::shared_ptr<BaseStreamDemuxer> streamDemuxer_; 334 std::shared_ptr<BaseStreamDemuxer> subStreamDemuxer_; 335 std::string bundleName_ {}; 336 std::string playerId_; 337 338 std::atomic<bool> isDecodeOptimizationEnabled_ {false}; 339 std::atomic<float> speed_ {1.0f}; 340 std::atomic<double> framerate_ {0.0}; 341 std::atomic<int32_t> decoderFramerateUpperLimit_ {DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT}; 342 343 std::string subtitlePluginName_; 344 std::shared_ptr<Plugins::DemuxerPlugin> subtitlePlugin_; 345 std::shared_ptr<MediaSource> subtitleMediaSource_; 346 bool isDump_ = false; 347 std::shared_ptr<DemuxerPluginManager> demuxerPluginManager_; 348 std::atomic<bool> isSelectBitRate_ = false; 349 uint32_t targetBitRate_ = 0; 350 std::string dumpPrefix_ = ""; 351 std::unordered_set<Plugins::MediaType> disabledMediaTracks_ {}; 352 353 std::unique_ptr<Task> parserRefInfoTask_; 354 bool isFirstParser_ = true; 355 bool isParserTaskEnd_ = false; 356 std::mutex parserTaskMutex_ {}; 357 int64_t duration_ {0}; 358 FileType fileType_ = FileType::UNKNOW; 359 360 std::mutex prerollMutex_ {}; 361 std::atomic<bool> inPreroll_ = false; 362 363 std::map<int32_t, int32_t> inSelectTrackType_{}; 364 std::map<int32_t, std::pair<int32_t, int32_t>> seekReadyStreamInfo_{}; 365 std::condition_variable rebootPluginCondition_; 366 std::atomic<bool> isSelectTrack_ = false; 367 std::atomic<bool> shouldCheckAudioFramePts_ = false; 368 int64_t lastAudioPts_ = 0; 369 std::atomic<bool> isOnEventNoMemory_ = false; 370 std::atomic<bool> isSeekError_ = false; 371 std::atomic<bool> shouldCheckSubtitleFramePts_ = false; 372 int64_t lastSubtitlePts_ = 0; 373 std::shared_ptr<VideoStreamReadyCallback> VideoStreamReadyCallback_ = nullptr; 374 std::mutex draggingMutex_ {}; 375 std::atomic<bool> isDemuxerLoopExecuting_ {false}; 376 std::atomic<bool> isFirstFrameAfterSeek_ {false}; 377 std::atomic<bool> isInterruptNeeded_ {false}; 378 bool isAutoMaintainPts_ = false; 379 std::map<uint32_t, std::shared_ptr<MaintainBaseInfo>> maintainBaseInfos_; 380 int64_t mediaStartPts_ {HST_TIME_NONE}; 381 bool isEnableReselectVideoTrack_ {false}; 382 uint32_t targetVideoTrackId_ {TRACK_ID_DUMMY}; 383 SyncFrameInfo syncFrameInfo_ {}; 384 std::mutex syncFrameInfoMutex_ {}; 385 bool perfRecEnabled_ { false }; 386 PerfRecorder perfRecorder_ {}; 387 int32_t apiVersion_ {0}; 388 bool isHlsFmp4_ {false}; 389 std::mutex changeStreamMutex_ {}; 390 }; 391 } // namespace Media 392 } // namespace OHOS 393 #endif // MEDIA_DEMUXER_H