1 /* 2 * Copyright (C) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef FFMPEG_DEMUXER_PLUGIN_H 17 #define FFMPEG_DEMUXER_PLUGIN_H 18 19 #include <atomic> 20 #include <vector> 21 #include <thread> 22 #include <map> 23 #include <queue> 24 #include <shared_mutex> 25 #include <list> 26 #include "buffer/avbuffer.h" 27 #include "plugin/demuxer_plugin.h" 28 #include "block_queue_pool.h" 29 #include "stream_parser_manager.h" 30 #include "reference_parser_manager.h" 31 #include "meta/meta.h" 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 #include "libavformat/avformat.h" 37 #include "libavcodec/avcodec.h" 38 #include "libavutil/dict.h" 39 #include "libavutil/opt.h" 40 #include "libavutil/parseutils.h" 41 #include "libavcodec/bsf.h" 42 #ifdef __cplusplus 43 } 44 #endif 45 46 namespace OHOS { 47 namespace Media { 48 namespace Plugins { 49 namespace Ffmpeg { 50 class FFmpegDemuxerPlugin : public DemuxerPlugin { 51 public: 52 explicit FFmpegDemuxerPlugin(std::string name); 53 ~FFmpegDemuxerPlugin() override; 54 Status Reset() override; 55 Status Start() override; 56 Status Stop() override; 57 Status Flush() override; 58 Status SetDataSource(const std::shared_ptr<DataSource>& source) override; 59 Status GetMediaInfo(MediaInfo& mediaInfo) override; 60 Status GetUserMeta(std::shared_ptr<Meta> meta) override; 61 Status SelectTrack(uint32_t trackId) override; 62 Status UnselectTrack(uint32_t trackId) override; 63 Status SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime) override; 64 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample) override; 65 Status GetNextSampleSize(uint32_t trackId, int32_t& size) override; 66 Status GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo) override; 67 void ResetEosStatus() override; 68 bool IsRefParserSupported() override; 69 Status ParserRefUpdatePos(int64_t timeStampMs, bool isForward = true) override; 70 Status ParserRefInfo() override; 71 Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo) override; 72 Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo) override; 73 Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo) override; 74 Status GetIFramePos(std::vector<uint32_t> &IFramePos) override; 75 Status Dts2FrameId(int64_t dts, uint32_t &frameId, bool offset = true) override; 76 77 Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex, 78 const uint64_t relativePresentationTimeUs, uint32_t &index) override; 79 Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex, 80 const uint32_t index, uint64_t &relativePresentationTimeUs) override; 81 void SetCacheLimit(uint32_t limitSize) override; 82 83 private: 84 enum DumpMode : unsigned long { 85 DUMP_NONE = 0, 86 DUMP_READAT_INPUT = 0b001, 87 DUMP_AVPACKET_OUTPUT = 0b010, 88 DUMP_AVBUFFER_OUTPUT = 0b100, 89 }; 90 enum IndexAndPTSConvertMode : unsigned int { 91 GET_FIRST_PTS, 92 INDEX_TO_RELATIVEPTS, 93 RELATIVEPTS_TO_INDEX, 94 GET_ALL_FRAME_PTS, 95 }; 96 struct IOContext { 97 std::shared_ptr<DataSource> dataSource {nullptr}; 98 int64_t offset {0}; 99 uint64_t fileSize {0}; 100 bool eos {false}; 101 std::atomic<bool> retry {false}; 102 uint32_t initDownloadDataSize {0}; 103 std::atomic<bool> initCompleted {false}; 104 DumpMode dumpMode {DUMP_NONE}; 105 }; 106 void ConvertCsdToAnnexb(const AVStream& avStream, Meta &format); 107 int64_t GetFileDuration(const AVFormatContext& avFormatContext); 108 int64_t GetStreamDuration(const AVStream& avStream); 109 110 static int AVReadPacket(void* opaque, uint8_t* buf, int bufSize); 111 static int AVWritePacket(void* opaque, uint8_t* buf, int bufSize); 112 static int64_t AVSeek(void* opaque, int64_t offset, int whence); 113 AVIOContext* AllocAVIOContext(int flags, IOContext *ioContext); 114 std::shared_ptr<AVFormatContext> InitAVFormatContext(IOContext *ioContext); 115 static int CheckContextIsValid(void* opaque, int &bufSize); 116 void NotifyInitializationCompleted(); 117 118 void InitParser(); 119 void InitBitStreamContext(const AVStream& avStream); 120 Status ConvertAvcToAnnexb(AVPacket& pkt); 121 Status PushEOSToAllCache(); 122 bool TrackIsSelected(const uint32_t trackId); 123 Status ReadPacketToCacheQueue(const uint32_t readId); 124 Status AddPacketToCacheQueue(AVPacket *pkt); 125 Status SetDrmCencInfo(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 126 void WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 127 Status ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 128 Status ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* avpacket, 129 std::shared_ptr<SamplePacket> dstSamplePacket); 130 Status SetEosSample(std::shared_ptr<AVBuffer> sample); 131 Status WriteBuffer(std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize); 132 void ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize, 133 std::multimap<std::string, std::vector<uint8_t>>& drmInfo); 134 bool GetNextFrame(const uint8_t *data, const uint32_t size); 135 bool NeedCombineFrame(uint32_t trackId); 136 AVPacket* CombinePackets(std::shared_ptr<SamplePacket> samplePacket); 137 Status ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 138 Status ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 139 Status GetSeiInfo(); 140 141 int FindNaluSpliter(int size, const uint8_t *data); 142 bool CanDropAvcPkt(const AVPacket& pkt); 143 bool CanDropHevcPkt(const AVPacket& pkt); 144 bool WebvttPktProcess(AVPacket *pkt); 145 bool IsWebvttMP4(const AVStream *avStream); 146 bool IsLessMaxReferenceParserFrames(uint32_t trackIndex); 147 void WebvttMP4EOSProcess(const AVPacket *pkt); 148 Status CheckCacheDataLimit(uint32_t trackId); 149 150 Status GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode, 151 uint32_t trackIndex, int64_t absolutePTS, uint32_t index); 152 Status PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode, 153 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 154 Status PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode, 155 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 156 void InitPTSandIndexConvert(); 157 void IndexToRelativePTSProcess(int64_t pts, uint32_t index); 158 void RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS); 159 void PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode, 160 int64_t pts, int64_t absolutePTS, uint32_t index); 161 int64_t absolutePTSIndexZero_ = INT64_MAX; 162 std::priority_queue<int64_t> indexToRelativePTSMaxHeap_; 163 uint32_t indexToRelativePTSFrameCount_ = 0; 164 uint32_t relativePTSToIndexPosition_ = 0; 165 int64_t relativePTSToIndexPTSMin_ = INT64_MAX; 166 int64_t relativePTSToIndexPTSMax_ = INT64_MIN; 167 int64_t relativePTSToIndexRightDiff_ = INT64_MAX; 168 int64_t relativePTSToIndexLeftDiff_ = INT64_MAX; 169 int64_t relativePTSToIndexTempDiff_ = INT64_MAX; 170 void ParserFirstDts(); 171 Status InitIoContext(); 172 Status ParserRefInit(); 173 Status ParserRefInfoLoop(AVPacket *pkt, uint32_t curStreamId); 174 Status SelectProGopId(); 175 void ParserBoxInfo(); 176 AVStream *GetVideoStream(); 177 178 std::mutex mutex_ {}; 179 std::shared_mutex sharedMutex_; 180 std::unordered_map<uint32_t, std::shared_ptr<std::mutex>> trackMtx_; 181 Seekable seekable_; 182 IOContext ioContext_; 183 std::vector<uint32_t> selectedTrackIds_; 184 BlockQueuePool cacheQueue_; 185 186 std::shared_ptr<AVInputFormat> pluginImpl_ {nullptr}; 187 std::shared_ptr<AVFormatContext> formatContext_ {nullptr}; 188 std::shared_ptr<AVBSFContext> avbsfContext_ {nullptr}; 189 std::shared_ptr<StreamParserManager> streamParser_ {nullptr}; 190 bool streamParserInited_ {false}; 191 192 Status GetVideoFirstKeyFrame(uint32_t trackIndex); 193 void ParseHEVCMetadataInfo(const AVStream& avStream, Meta &format); 194 AVPacket *firstFrame_ = nullptr; 195 196 std::atomic<bool> parserState_ = true; 197 IOContext parserRefIoContext_; 198 std::shared_ptr<AVFormatContext> parserRefFormatContext_{nullptr}; 199 int parserRefVideoStreamIdx_ = -1; 200 std::shared_ptr<ReferenceParserManager> referenceParser_{nullptr}; 201 int32_t parserCurGopId_ = 0; 202 int64_t pendingSeekMsTime_ = -1; 203 int64_t parserRefStartTime_ = -1; 204 std::list<uint32_t> processingIFrame_; 205 std::vector<uint32_t> IFramePos_; 206 double fps_{0}; 207 int64_t firstDts_ = 0; 208 uint32_t dtsOffset_ = 0; 209 bool isSdtpExist_ = false; 210 std::mutex syncMutex_; 211 bool updatePosIsForward_ = true; 212 bool isInit_ = false; 213 uint32_t cachelimitSize_ = 0; 214 bool outOfLimit_ = false; 215 bool setLimitByUser = false; 216 217 // dfx 218 struct TrackDfxInfo { 219 int frameIndex = 0; // for each track 220 int64_t lastPts; 221 int64_t lastPos; 222 int64_t lastDurantion; 223 }; 224 struct DumpParam { 225 DumpMode mode; 226 uint8_t* buf; 227 int trackId; 228 int64_t offset; 229 int size; 230 int index; 231 int64_t pts; 232 int64_t pos; 233 }; 234 std::unordered_map<int, TrackDfxInfo> trackDfxInfoMap_; 235 DumpMode dumpMode_ {DUMP_NONE}; 236 static std::atomic<int> readatIndex_; 237 int avpacketIndex_ {0}; 238 239 static void Dump(const DumpParam &dumpParam); 240 241 std::vector<int64_t> ptsListOrg_; 242 std::vector<int64_t> ptsListFromZero_; 243 std::unordered_map<int32_t, int64_t> iFramePtsMap_; 244 Status GetGopIdFromSeekPos(int64_t seekMs, int32_t &gopId); 245 Status ParserRefCheckVideoValid(const AVStream *videoStream); 246 }; 247 } // namespace Ffmpeg 248 } // namespace Plugins 249 } // namespace Media 250 } // namespace OHOS 251 #endif // FFMPEG_DEMUXER_PLUGIN_H 252