1 /* 2 * Copyright (C) 2023-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef FFMPEG_DEMUXER_PLUGIN_H 17 #define FFMPEG_DEMUXER_PLUGIN_H 18 19 #include <atomic> 20 #include <vector> 21 #include <thread> 22 #include <map> 23 #include <queue> 24 #include <shared_mutex> 25 #include <list> 26 #include "buffer/avbuffer.h" 27 #include "plugin/demuxer_plugin.h" 28 #include "block_queue_pool.h" 29 #include "multi_stream_parser_manager.h" 30 #include "reference_parser_manager.h" 31 #include "meta/meta.h" 32 #include "qos.h" 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 #include "libavformat/avformat.h" 38 #include "libavcodec/avcodec.h" 39 #include "libavutil/dict.h" 40 #include "libavutil/opt.h" 41 #include "libavutil/parseutils.h" 42 #include "libavcodec/bsf.h" 43 #ifdef __cplusplus 44 } 45 #endif 46 47 namespace OHOS { 48 namespace Media { 49 namespace Plugins { 50 namespace Ffmpeg { 51 extern const std::vector<AVCodecID> g_streamContainedXPS; 52 class FFmpegDemuxerPlugin : public DemuxerPlugin { 53 public: 54 explicit FFmpegDemuxerPlugin(std::string name); 55 ~FFmpegDemuxerPlugin() override; 56 Status Reset() override; 57 Status Start() override; 58 Status Stop() override; 59 Status Flush() override; 60 Status SetDataSource(const std::shared_ptr<DataSource>& source) override; 61 Status GetMediaInfo(MediaInfo& mediaInfo) override; 62 Status GetUserMeta(std::shared_ptr<Meta> meta) override; 63 Status SelectTrack(uint32_t trackId) override; 64 Status UnselectTrack(uint32_t trackId) override; 65 Status SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime) override; 66 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample) override; 67 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample, uint32_t timeout) override; 68 Status GetNextSampleSize(uint32_t trackId, int32_t& size) override; 69 Status GetNextSampleSize(uint32_t trackId, int32_t& size, uint32_t timeout) override; 70 Status Pause() override; 71 Status GetLastPTSByTrackId(uint32_t trackId, int64_t &lastPTS) override; 72 Status GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo) override; 73 void ResetEosStatus() override; 74 bool IsRefParserSupported() override; 75 Status ParserRefUpdatePos(int64_t timeStampMs, bool isForward = true) override; 76 Status ParserRefInfo() override; 77 Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo) override; 78 Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo) override; 79 Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo) override; 80 Status GetIFramePos(std::vector<uint32_t> &IFramePos) override; 81 Status Dts2FrameId(int64_t dts, uint32_t &frameId) override; 82 Status SeekMs2FrameId(int64_t seekMs, uint32_t &frameId) override; 83 Status FrameId2SeekMs(uint32_t frameId, int64_t &seekMs) override; 84 Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex, 85 const uint64_t relativePresentationTimeUs, uint32_t &index) override; 86 Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex, 87 const uint32_t index, uint64_t &relativePresentationTimeUs) override; 88 void SetCacheLimit(uint32_t limitSize) override; 89 Status GetCurrentCacheSize(uint32_t trackId, uint32_t& size) override; 90 bool GetProbeSize(int32_t &offset, int32_t &size) override; 91 void SetInterruptState(bool isInterruptNeeded) override; 92 Status SetDataSourceWithProbSize(const std::shared_ptr<DataSource>& source, 93 const int32_t probSize) override; 94 Status BoostReadThreadPriority() override; 95 private: 96 enum ThreadState : unsigned int { 97 NOT_STARTED, 98 WAITING, 99 READING, 100 }; 101 102 enum InvokerType : unsigned int { 103 INVOKER_NONE = 0, 104 INIT, 105 FLUSH, 106 READ, 107 SEEK, 108 DESTORY, 109 }; 110 111 enum DumpMode : unsigned long { 112 DUMP_NONE = 0, 113 DUMP_READAT_INPUT = 0b001, 114 DUMP_AVPACKET_OUTPUT = 0b010, 115 DUMP_AVBUFFER_OUTPUT = 0b100, 116 }; 117 enum IndexAndPTSConvertMode : unsigned int { 118 GET_FIRST_PTS, 119 INDEX_TO_RELATIVEPTS, 120 RELATIVEPTS_TO_INDEX, 121 GET_ALL_FRAME_PTS, 122 }; 123 struct IOContext { 124 std::shared_ptr<DataSource> dataSource {nullptr}; 125 int64_t offset {0}; 126 uint64_t fileSize {0}; 127 bool eos {false}; 128 std::atomic<bool> retry {false}; 129 uint32_t initDownloadDataSize {0}; 130 std::atomic<bool> initCompleted {false}; 131 DumpMode dumpMode {DUMP_NONE}; 132 bool isLimit {false}; 133 bool isLimitType {false}; 134 int32_t sizeLimit {0}; 135 int32_t readSizeCnt {0}; 136 std::atomic<bool> initErrorAgain {false}; 137 std::mutex invokerTypeMutex; 138 std::atomic<InvokerType> invokerType {INVOKER_NONE}; 139 bool readCbReady {false}; 140 }; 141 142 bool SelectedVideo(); 143 bool NeedDropAfterSeek(uint32_t trackId, int64_t pts); 144 std::atomic<int64_t> seekTime_ = AV_NOPTS_VALUE; 145 std::atomic<SeekMode> seekMode_ = SeekMode::SEEK_NEXT_SYNC; 146 void ConvertCsdToAnnexb(const AVStream& avStream, Meta &format); 147 int64_t GetFileDuration(const AVFormatContext& avFormatContext); 148 int64_t GetStreamDuration(const AVStream& avStream); 149 150 bool FrameReady(Status ret); 151 int SelectSeekTrack() const; 152 Status CheckSeekParams(int64_t seekTime, SeekMode mode) const; 153 void SyncSeekThread(); 154 Status DoSeekInternal(int trackIndex, int64_t seekTime, int64_t ffTime, SeekMode mode, int64_t& realSeekTime); 155 bool IsUseFirstFrameDts(int trackIndex, int64_t seekTime); 156 void GetStreamInitialParams(); 157 void SetStreamInitialParams(uint32_t trackId, Meta &format); 158 159 static int AVReadPacket(void* opaque, uint8_t* buf, int bufSize); 160 static int HandleReadOK(IOContext* ioContext, int dataSize); 161 static int HandleReadAgain(IOContext* ioContext, int dataSize, int& tryCount); 162 static int HandleReadEOS(IOContext* ioContext); 163 static int HandleReadError(int result); 164 static void UpdateInitDownloadData(IOContext* ioContext, int dataSize); 165 static int AVWritePacket(void* opaque, uint8_t* buf, int bufSize); 166 static int64_t AVSeek(void* opaque, int64_t offset, int whence); 167 AVIOContext* AllocAVIOContext(int flags, IOContext *ioContext); 168 std::shared_ptr<AVFormatContext> InitAVFormatContext(IOContext *ioContext); 169 static int CheckContextIsValid(void* opaque, int &bufSize); 170 void NotifyInitializationCompleted(); 171 172 void InitParser(); 173 void InitBitStreamContext(const AVStream& avStream); 174 Status ConvertAvcToAnnexb(AVPacket& pkt); 175 Status PushEOSToAllCache(); 176 bool TrackIsSelected(const uint32_t trackId); 177 Status ReadPacketToCacheQueue(const uint32_t readId); 178 Status AddPacketToCacheQueue(AVPacket *pkt); 179 Status SetDrmCencInfo(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 180 void WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 181 Status BufferIsValid(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 182 Status ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 183 Status ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* avpacket, 184 std::shared_ptr<SamplePacket> dstSamplePacket); 185 Status SetEosSample(std::shared_ptr<AVBuffer> sample); 186 Status WriteBuffer(std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, uint32_t writeSize); 187 void ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize, 188 std::multimap<std::string, std::vector<uint8_t>>& drmInfo); 189 bool NeedCombineFrame(uint32_t trackId); 190 AVPacket* CombinePackets(std::shared_ptr<SamplePacket> samplePacket); 191 Status ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 192 Status ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 193 bool HasCodecParameters(); 194 Status GetMediaInfo(); 195 void ResetParam(); 196 197 bool WebvttPktProcess(AVPacket *pkt); 198 bool IsWebvttMP4(const AVStream *avStream); 199 bool IsLessMaxReferenceParserFrames(uint32_t trackIndex); 200 void WebvttMP4EOSProcess(const AVPacket *pkt); 201 Status CheckCacheDataLimit(uint32_t trackId); 202 203 Status GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode, 204 uint32_t trackIndex, int64_t absolutePTS, uint32_t index); 205 Status PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode, 206 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 207 Status PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode, 208 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 209 void InitPTSandIndexConvert(); 210 void IndexToRelativePTSProcess(int64_t pts, uint32_t index); 211 void RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS); 212 void PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode, 213 int64_t pts, int64_t absolutePTS, uint32_t index, int64_t dts); 214 void ResetContext(); 215 int64_t absolutePTSIndexZero_ = INT64_MAX; 216 std::priority_queue<int64_t> indexToRelativePTSMaxHeap_; 217 uint32_t indexToRelativePTSFrameCount_ = 0; 218 uint32_t relativePTSToIndexPosition_ = 0; 219 int64_t relativePTSToIndexPTSMin_ = INT64_MAX; 220 int64_t relativePTSToIndexPTSMax_ = INT64_MIN; 221 int64_t relativePTSToIndexRightDiff_ = INT64_MAX; 222 int64_t relativePTSToIndexLeftDiff_ = INT64_MAX; 223 int64_t relativePTSToIndexTempDiff_ = INT64_MAX; 224 Status InitIoContext(); 225 void InitIoContextInDemuxer(const std::shared_ptr<DataSource>& source); 226 Status ParserRefInit(); 227 Status ParserRefInfoLoop(AVPacket *pkt, uint32_t curStreamId); 228 Status SelectProGopId(); 229 void ParserBoxInfo(); 230 AVStream *GetVideoStream(); 231 232 std::mutex mutex_ {}; 233 std::shared_mutex sharedMutex_; 234 std::unordered_map<uint32_t, std::shared_ptr<std::mutex>> trackMtx_; 235 Seekable seekable_; 236 IOContext ioContext_; 237 std::vector<uint32_t> selectedTrackIds_; 238 BlockQueuePool cacheQueue_; 239 MediaInfo mediaInfo_; 240 FileType fileType_ = FileType::UNKNOW; 241 242 std::shared_ptr<AVInputFormat> pluginImpl_ {nullptr}; 243 std::shared_ptr<AVFormatContext> formatContext_ {nullptr}; 244 std::map<uint32_t, std::shared_ptr<AVBSFContext>> avbsfContexts_ {}; 245 246 void UpdateReferenceIds(); 247 std::map<int32_t, std::vector<int32_t>> referenceIdsMap_ {}; 248 249 Status ParseVideoFirstFrames(); 250 bool AllVideoFirstFramesReady(); 251 bool AllSupportTrackFramesReady(); 252 Status SetVideoFirstFrame(AVPacket* pkt, bool isConvert = true); 253 bool VideoFirstFrameValid(uint32_t trackIndex); 254 std::map<int32_t, AVPacket *> videoFirstFrameMap_ {}; 255 std::unordered_map<int32_t, int64_t> seekCalibMap_ {}; 256 bool TrackIsChecked(const uint32_t trackId); 257 std::vector<uint32_t> checkedTrackIds_ {}; 258 void ClearUnselectTrackCache(); 259 bool needClear_ = true; 260 261 std::shared_ptr<MultiStreamParserManager> streamParsers_ {nullptr}; 262 263 void ParseHEVCMetadataInfo(const AVStream& avStream, Meta &format); 264 std::atomic<bool> parserState_ = true; 265 IOContext parserRefIoContext_; 266 std::shared_ptr<AVFormatContext> parserRefCtx_{nullptr}; 267 int parserRefIdx_ = -1; 268 std::shared_ptr<ReferenceParserManager> referenceParser_{nullptr}; 269 int32_t parserCurGopId_ = 0; 270 int64_t pendingSeekMsTime_ = -1; 271 int64_t parserRefStartTime_ = -1; 272 std::list<uint32_t> processingIFrame_; 273 std::vector<uint32_t> IFramePos_; 274 int64_t minPts_ = 0; 275 int64_t startPts_ = 0; 276 uint32_t ptsCnt_ = 0; 277 bool isSdtpExist_ = false; 278 std::mutex syncMutex_; 279 bool updatePosIsForward_ = true; 280 bool isInit_ = false; 281 uint32_t cachelimitSize_ = 0; 282 bool outOfLimit_ = false; 283 bool setLimitByUser = false; 284 std::atomic<bool> isInterruptNeeded_{false}; 285 286 // dfx 287 struct TrackDfxInfo { 288 int frameIndex = 0; // for each track 289 int64_t lastPts; 290 int64_t lastPos; 291 int64_t lastDuration; 292 bool dumpFirstInfo = false; 293 }; 294 enum Stage : int32_t { 295 FIRST_READ = 0, 296 FILE_END = 1, 297 }; 298 void UpdateLastPacketInfo(int32_t trackId, int64_t pts, int64_t pos, int64_t duration); 299 void DumpPacketInfo(int32_t trackId, Stage stage); 300 struct DumpParam { 301 DumpMode mode; 302 uint8_t* buf; 303 int trackId; 304 int64_t offset; 305 int size; 306 int index; 307 int64_t pts; 308 int64_t pos; 309 }; 310 std::unordered_map<int, TrackDfxInfo> trackDfxInfoMap_; 311 DumpMode dumpMode_ {DUMP_NONE}; 312 static std::atomic<int> readatIndex_; 313 int avpacketIndex_ {0}; 314 315 static void Dump(const DumpParam &dumpParam); 316 317 std::map<int64_t, int64_t> pts2DtsMap_; 318 std::unordered_map<int32_t, int64_t> iFramePtsMap_; 319 Status GetGopIdFromSeekPos(int64_t seekMs, int32_t &gopId); 320 Status ParserRefCheckVideoValid(const AVStream *videoStream); 321 bool IsMultiVideoTrack(); 322 int AVReadFrameLimit(AVPacket *pkt); 323 Status SetAVReadFrameLimit(); 324 325 Status WaitForLoop(const uint32_t trackId, const uint32_t timeout); 326 bool ShouldWaitForRead(uint32_t trackId); 327 void FFmpegReadLoop(); 328 bool NeedWaitForRead(); 329 void HandleReadWait(); 330 bool EnsurePacketAllocated(AVPacket*& pkt); 331 bool ReadAndProcessFrame(AVPacket* pkt); 332 void HandleAVPacketEndOfStream(AVPacket* pkt); 333 void HandleAVPacketReadError(AVPacket* pkt, int ffmpegRet); 334 bool ReadOnePacketAndProcessWebVTT(AVPacket* pkt); 335 void ReleaseFFmpegReadLoop(); 336 std::unique_ptr<std::thread> readThread_ {nullptr}; 337 std::condition_variable readLoopCv_; 338 static std::condition_variable readCbCv_; 339 std::condition_variable readCacheCv_; 340 static std::mutex readPacketMutex_; 341 std::mutex getNextSampleMutex_; 342 std::mutex readSampleMutex_; 343 std::mutex fFmpegReadLoopMutex_; 344 uint32_t trackId_ = 0; 345 ThreadState threadState_ {ThreadState::NOT_STARTED}; 346 std::atomic<Status> readLoopStatus_ = {Status::OK}; 347 bool isPauseReadPacket_ = false; 348 std::unordered_map<int, int> readModeMap_; // 0 mean sync read, 1 mean async read 349 std::mutex seekWaitMutex_; 350 std::condition_variable seekWaitCv_; 351 std::atomic<bool> threadReady_ {false}; 352 std::atomic<bool> isWaitingForReadThread_ {false}; 353 std::unordered_map<uint32_t, Meta> streamInitialParam_; 354 355 std::atomic<bool> isAsyncReadThreadPrioritySet_ = false; 356 void UpdateAsyncReadThreadPriority(); 357 }; 358 359 typedef struct DtsFinder { DtsFinderDtsFinder360 explicit DtsFinder(int64_t dts) : dts_(dts) { } operatorDtsFinder361 bool operator ()(const std::map<int64_t, int64_t>::value_type &item) 362 { 363 return dts_ == item.second || dts_ == item.second - 1; 364 } 365 int64_t dts_; 366 } DtsFinder; 367 } // namespace Ffmpeg 368 } // namespace Plugins 369 } // namespace Media 370 } // namespace OHOS 371 #endif // FFMPEG_DEMUXER_PLUGIN_H 372