1 /* 2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef MEDIA_DEMUXER_H 17 #define MEDIA_DEMUXER_H 18 19 #include <atomic> 20 #include <limits> 21 #include <string> 22 #include <shared_mutex> 23 #include <unordered_set> 24 25 #include "osal/task/condition_variable.h" 26 #include "avcodec_common.h" 27 #include "buffer/avbuffer.h" 28 #include "common/media_source.h" 29 #include "common/seek_callback.h" 30 #include "demuxer/type_finder.h" 31 #include "filter/filter.h" 32 #include "meta/media_types.h" 33 #include "osal/task/autolock.h" 34 #include "osal/task/task.h" 35 #include "plugin/plugin_base.h" 36 #include "plugin/plugin_info.h" 37 #include "plugin/plugin_time.h" 38 #include "plugin/demuxer_plugin.h" 39 #include "performance_utils.h" 40 41 namespace OHOS { 42 namespace Media { 43 namespace { 44 constexpr uint32_t TRACK_ID_DUMMY = std::numeric_limits<uint32_t>::max(); 45 constexpr int32_t DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT = 120; 46 } 47 48 using MediaSource = OHOS::Media::Plugins::MediaSource; 49 using FileType = OHOS::Media::Plugins::FileType; 50 class BaseStreamDemuxer; 51 class DemuxerPluginManager; 52 class Source; 53 54 class AVBufferQueueProducer; 55 56 class MediaDemuxer : public std::enable_shared_from_this<MediaDemuxer>, public Plugins::Callback { 57 public: 58 explicit MediaDemuxer(); 59 ~MediaDemuxer() override; 60 61 Status SetDataSource(const std::shared_ptr<MediaSource> &source); 62 Status SetSubtitleSource(const std::shared_ptr<MediaSource> &source); 63 void SetBundleName(const std::string& bundleName); 64 Status SetOutputBufferQueue(int32_t trackId, const sptr<AVBufferQueueProducer>& producer); 65 66 std::shared_ptr<Meta> GetGlobalMetaInfo() const; 67 std::vector<std::shared_ptr<Meta>> GetStreamMetaInfo() const; 68 std::shared_ptr<Meta> GetUserMeta(); 69 70 Status SeekTo(int64_t seekTime, Plugins::SeekMode mode, int64_t& realSeekTime); 71 Status Reset(); 72 Status Start(); 73 Status Stop(); 74 Status Pause(); 75 Status PauseDragging(); 76 Status PauseAudioAlign(); 77 Status Resume(); 78 Status ResumeDragging(); 79 Status ResumeAudioAlign(); 80 Status Flush(); 81 Status Preroll(); 82 Status PausePreroll(); 83 84 Status StartTask(int32_t trackId); 85 Status SelectTrack(int32_t trackId); 86 Status UnselectTrack(int32_t trackId); 87 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample); 88 Status GetBitRates(std::vector<uint32_t> &bitRates); 89 Status SelectBitRate(uint32_t bitRate); 90 Status GetDownloadInfo(DownloadInfo& downloadInfo); 91 Status GetPlaybackInfo(PlaybackInfo& playbackInfo); 92 Status GetMediaKeySystemInfo(std::multimap<std::string, std::vector<uint8_t>> &infos); 93 void SetDrmCallback(const std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> &callback); 94 void OnEvent(const Plugins::PluginEvent &event) override; 95 void OnSeekReadyEvent(const Plugins::PluginEvent &event); 96 void OnDfxEvent(const Plugins::PluginDfxEvent &event) override; 97 98 Status SetPerfRecEnabled(bool isPerfRecEnabled); 99 100 std::map<uint32_t, sptr<AVBufferQueueProducer>> GetBufferQueueProducerMap(); 101 Status PauseTaskByTrackId(int32_t trackId); 102 bool IsRenderNextVideoFrameSupported(); 103 104 void SetEventReceiver(const std::shared_ptr<Pipeline::EventReceiver> &receiver); 105 bool GetDuration(int64_t& durationMs); 106 void SetPlayerId(const std::string &playerId); 107 void SetInterruptState(bool isInterruptNeeded); 108 void SetDumpInfo(bool isDump, uint64_t instanceId); 109 110 Status OptimizeDecodeSlow(bool isDecodeOptimizationEnabled); 111 Status SetDecoderFramerateUpperLimit(int32_t decoderFramerateUpperLimit, uint32_t trackId); 112 Status SetSpeed(float speed); 113 Status SetFrameRate(double frameRate, uint32_t trackId); 114 115 bool IsLocalDrmInfosExisted(); 116 Status DisableMediaTrack(Plugins::MediaType mediaType); 117 void OnBufferAvailable(uint32_t trackId); 118 void SetSelectBitRateFlag(bool flag, uint32_t desBitRate) override; 119 bool CanAutoSelectBitRate() override; 120 void OnDumpInfo(int32_t fd); 121 122 bool IsRefParserSupported(); 123 Status StartReferenceParser(int64_t startTimeMs, bool isForward = true); 124 Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo); 125 Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo); 126 Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo); 127 bool IsVideoEos(); 128 bool HasEosTrack(); 129 Status GetIFramePos(std::vector<uint32_t> &IFramePos); 130 Status Dts2FrameId(int64_t dts, uint32_t &frameId, bool offset = true); 131 void RegisterVideoStreamReadyCallback(const std::shared_ptr<VideoStreamReadyCallback> &callback); 132 void DeregisterVideoStreamReadyCallback(); 133 134 Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex, 135 const uint64_t relativePresentationTimeUs, uint32_t &index); 136 Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex, 137 const uint32_t index, uint64_t &relativePresentationTimeUs); 138 139 Status ResumeDemuxerReadLoop(); 140 Status PauseDemuxerReadLoop(); 141 void SetCacheLimit(uint32_t limitSize); 142 void SetEnableOnlineFdCache(bool isEnableFdCache); 143 void WaitForBufferingEnd(); 144 int32_t GetCurrentVideoTrackId(); 145 uint32_t GetTargetVideoTrackId(std::vector<std::shared_ptr<Meta>> trackInfos); 146 void SetIsEnableReselectVideoTrack(bool isEnable); 147 bool IsHasMultiVideoTrack(); 148 void SetApiVersion(int32_t apiVersion); 149 private: 150 class AVBufferQueueProducerListener; 151 class TrackWrapper; 152 153 struct MediaMetaData { 154 std::vector<std::shared_ptr<Meta>> trackMetas; 155 std::shared_ptr<Meta> globalMeta; 156 }; 157 158 struct MaintainBaseInfo { 159 int64_t segmentOffset = -1; 160 int64_t basePts = -1; 161 int64_t lastPts = 0; 162 }; 163 164 struct SyncFrameInfo { 165 int64_t pts = -1; 166 int32_t skipOpenGopUnrefFrameCnt = 0; 167 }; 168 169 bool isHttpSource_ = false; 170 std::string videoMime_{}; 171 172 void InitMediaMetaData(const Plugins::MediaInfo& mediaInfo); 173 void InitDefaultTrack(const Plugins::MediaInfo& mediaInfo, uint32_t& videoTrackId, 174 uint32_t& audioTrackId, uint32_t& subtitleTrackId, std::string& videoMime); 175 std::shared_ptr<Meta> GetTrackMeta(uint32_t trackId); 176 Status AddDemuxerCopyTask(uint32_t trackId, TaskType type); 177 178 Status StopAllTask(); 179 Status PauseAllTask(); 180 Status ResumeAllTask(); 181 void AccelerateTrackTask(uint32_t trackId); 182 void SetTrackNotifyFlag(uint32_t trackId, bool isNotifyNeeded); 183 void ResetInner(); 184 185 bool GetDrmInfosUpdated(const std::multimap<std::string, std::vector<uint8_t>> &newInfos, 186 std::multimap<std::string, std::vector<uint8_t>> &result); 187 Status ProcessDrmInfos(); 188 void HandleSourceDrmInfoEvent(const std::multimap<std::string, std::vector<uint8_t>> &info); 189 Status ReportDrmInfos(const std::multimap<std::string, std::vector<uint8_t>> &info); 190 191 bool HasVideo(); 192 void DumpBufferToFile(uint32_t trackId, std::shared_ptr<AVBuffer> buffer); 193 bool IsBufferDroppable(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 194 void CheckDropAudioFrame(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 195 bool IsTrackDisabled(Plugins::MediaType mediaType); 196 bool CheckTrackEnabledById(uint32_t trackId); 197 bool HandleDashChangeStream(uint32_t trackId); 198 199 Status SeekToTimeAfter(); 200 bool SelectBitRateChangeStream(uint32_t trackId); 201 bool SelectTrackChangeStream(uint32_t trackId); 202 bool HandleSelectTrackChangeStream(int32_t trackId, int32_t newStreamID, int32_t& newTrackId); 203 std::shared_ptr<Plugins::DemuxerPlugin> GetCurFFmpegPlugin(); 204 void UpdateThreadPriority(uint32_t trackId); 205 206 Plugins::Seekable seekable_; 207 Plugins::Seekable subSeekable_; 208 std::string uri_; 209 std::string SubtitleUri_; 210 uint64_t mediaDataSize_ = 0; 211 uint64_t subMediaDataSize_; 212 213 std::shared_ptr<MediaSource> mediaSource_; 214 std::shared_ptr<Source> source_; 215 std::shared_ptr<Source> subtitleSource_; 216 MediaMetaData mediaMetaData_; 217 218 int64_t ReadLoop(uint32_t trackId); 219 Status CopyFrameToUserQueue(uint32_t trackId); 220 bool GetBufferFromUserQueue(uint32_t queueIndex, uint32_t size = 0); 221 Status InnerReadSample(uint32_t trackId, std::shared_ptr<AVBuffer>); 222 Status InnerSelectTrack(int32_t trackId); 223 Status HandleReadSample(uint32_t trackId); 224 int64_t ParserRefInfo(); 225 void TryRecvParserTask(); 226 227 Status HandleSelectTrack(int32_t trackId); 228 Status HandleDashSelectTrack(int32_t trackId); 229 Status DoSelectTrack(int32_t trackId, int32_t curTrackId); 230 Status HandleRebootPlugin(int32_t trackId, bool& isRebooted); 231 bool DashCheckChangeStream(uint32_t trackId); 232 233 bool IsSubtitleMime(const std::string& mime); 234 Status InnerPrepare(); 235 Status HandleAutoMaintainPts(uint32_t trackId, std::shared_ptr<AVBuffer> sample); 236 Status InitPtsInfo(); 237 void UpdateBufferQueueListener(int32_t trackId); 238 bool IsOpenGopBufferDroppable(std::shared_ptr<AVBuffer> sample, uint32_t trackId); 239 void UpdateSyncFrameInfo(std::shared_ptr<AVBuffer> sample, uint32_t trackId, bool isDiscardable = false); 240 void EnterDraggingOpenGopCnt(); 241 void ResetDraggingOpenGopCnt(); 242 Status ReadSampleWithPerfRecord(const std::shared_ptr<Plugins::DemuxerPlugin> &pluginTemp, 243 const int32_t &innerTrackID, const std::shared_ptr<AVBuffer> &sample); 244 Status HandleTrackEos(uint32_t trackId); 245 void SetOutputBufferPts(std::shared_ptr<AVBuffer> &outputBuffer); 246 247 Mutex mapMutex_{}; 248 std::map<uint32_t, std::shared_ptr<TrackWrapper>> trackMap_; 249 std::map<uint32_t, sptr<AVBufferQueueProducer>> bufferQueueMap_; 250 std::map<uint32_t, std::shared_ptr<AVBuffer>> bufferMap_; 251 std::map<uint32_t, bool> eosMap_; 252 std::map<uint32_t, uint32_t> requestBufferErrorCountMap_; 253 std::atomic<bool> isThreadExit_ = true; 254 bool useBufferQueue_ = false; 255 bool isAccurateSeekForHLS_ = false; 256 int64_t videoStartTime_{0}; 257 258 std::shared_mutex drmMutex{}; 259 std::mutex isSelectTrackMutex_{}; 260 std::mutex rebootPluginMutex_{}; 261 std::multimap<std::string, std::vector<uint8_t>> localDrmInfos_; 262 std::shared_ptr<OHOS::MediaAVCodec::AVDemuxerCallback> drmCallback_; 263 264 std::map<uint32_t, std::unique_ptr<Task>> taskMap_; 265 std::shared_ptr<Pipeline::EventReceiver> eventReceiver_; 266 int64_t lastSeekTime_{Plugins::HST_TIME_NONE}; 267 bool isSeeked_{false}; 268 uint32_t videoTrackId_{TRACK_ID_DUMMY}; 269 270 uint32_t audioTrackId_{TRACK_ID_DUMMY}; 271 uint32_t subtitleTrackId_{TRACK_ID_DUMMY}; 272 bool firstAudio_{true}; 273 274 std::atomic<bool> isStopped_ = true; 275 std::atomic<bool> isPaused_ = false; 276 std::shared_ptr<BaseStreamDemuxer> streamDemuxer_; 277 std::shared_ptr<BaseStreamDemuxer> subStreamDemuxer_; 278 std::string bundleName_ {}; 279 280 std::string playerId_; 281 int64_t duration_ {0}; 282 FileType fileType_ = FileType::UNKNOW; 283 284 std::atomic<bool> isDecodeOptimizationEnabled_ {false}; 285 std::atomic<float> speed_ {1.0f}; 286 std::atomic<double> framerate_ {0.0}; 287 std::atomic<int32_t> decoderFramerateUpperLimit_ {DEFAULT_DECODE_FRAMERATE_UPPER_LIMIT}; 288 std::unordered_set<Plugins::MediaType> disabledMediaTracks_ {}; 289 std::string subtitlePluginName_; 290 std::shared_ptr<Plugins::DemuxerPlugin> subtitlePlugin_; 291 std::shared_ptr<MediaSource> subtitleMediaSource_; 292 bool isDump_ = false; 293 std::shared_ptr<DemuxerPluginManager> demuxerPluginManager_; 294 std::atomic<bool> isSelectBitRate_ = false; 295 uint32_t targetBitRate_ = 0; 296 std::string dumpPrefix_ = ""; 297 298 std::unique_ptr<Task> parserRefInfoTask_; 299 bool isFirstParser_ = true; 300 bool isParserTaskEnd_ = false; 301 std::atomic<bool> isOnEventNoMemory_ = false; 302 std::atomic<bool> isSeekError_ = false; 303 std::shared_ptr<VideoStreamReadyCallback> VideoStreamReadyCallback_ = nullptr; 304 std::mutex draggingMutex_ {}; 305 306 std::atomic<bool> isDemuxerLoopExecuting_ {false}; 307 std::atomic<bool> isFirstFrameAfterSeek_ {false}; 308 309 std::mutex prerollMutex_ {}; 310 std::atomic<bool> inPreroll_ = false; 311 312 std::map<int32_t, int32_t> inSelectTrackType_{}; 313 std::map<int32_t, std::pair<int32_t, int32_t>> seekReadyStreamInfo_{}; 314 std::condition_variable rebootPluginCondition_; 315 std::atomic<bool> isSelectTrack_ = false; 316 std::atomic<bool> shouldCheckAudioFramePts_ = false; 317 int64_t lastAudioPts_ = 0; 318 std::atomic<bool> shouldCheckSubtitleFramePts_ = false; 319 int64_t lastSubtitlePts_ = 0; 320 std::atomic<bool> isInterruptNeeded_ {false}; 321 bool isAutoMaintainPts_ = false; 322 std::map<uint32_t, std::shared_ptr<MaintainBaseInfo>> maintainBaseInfos_; 323 bool isEnableReselectVideoTrack_ {false}; 324 int32_t videoTrackCount_ = 0; 325 uint32_t targetVideoTrackId_ {TRACK_ID_DUMMY}; 326 SyncFrameInfo syncFrameInfo_ {}; 327 std::mutex syncFrameInfoMutex_ {}; 328 bool perfRecEnabled_ { false }; 329 PerfRecorder perfRecorder_ {}; 330 int32_t apiVersion_ {0}; 331 }; 332 } // namespace Media 333 } // namespace OHOS 334 #endif // MEDIA_DEMUXER_H 335