1 /* 2 * Copyright (C) 2023 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef FFMPEG_DEMUXER_PLUGIN_H 17 #define FFMPEG_DEMUXER_PLUGIN_H 18 19 #include <atomic> 20 #include <vector> 21 #include <thread> 22 #include <map> 23 #include <queue> 24 #include <shared_mutex> 25 #include <list> 26 #include "buffer/avbuffer.h" 27 #include "plugin/demuxer_plugin.h" 28 #include "block_queue_pool.h" 29 #include "stream_parser_manager.h" 30 #include "reference_parser_manager.h" 31 #include "meta/meta.h" 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 #include "libavformat/avformat.h" 37 #include "libavcodec/avcodec.h" 38 #include "libavutil/dict.h" 39 #include "libavutil/opt.h" 40 #include "libavutil/parseutils.h" 41 #include "libavcodec/bsf.h" 42 #ifdef __cplusplus 43 } 44 #endif 45 46 namespace OHOS { 47 namespace Media { 48 namespace Plugins { 49 namespace Ffmpeg { 50 class FFmpegDemuxerPlugin : public DemuxerPlugin { 51 public: 52 explicit FFmpegDemuxerPlugin(std::string name); 53 ~FFmpegDemuxerPlugin() override; 54 Status Reset() override; 55 Status Start() override; 56 Status Stop() override; 57 Status Flush() override; 58 Status SetDataSource(const std::shared_ptr<DataSource>& source) override; 59 Status GetMediaInfo(MediaInfo& mediaInfo) override; 60 Status GetUserMeta(std::shared_ptr<Meta> meta) override; 61 Status SelectTrack(uint32_t trackId) override; 62 Status UnselectTrack(uint32_t trackId) override; 63 Status SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime) override; 64 Status ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample) override; 65 Status GetNextSampleSize(uint32_t trackId, int32_t& size) override; 66 Status GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo) override; 67 void ResetEosStatus() override; 68 Status ParserRefUpdatePos(int64_t timeStampMs, bool isForward = true) override; 69 Status ParserRefInfo() override; 70 Status GetFrameLayerInfo(std::shared_ptr<AVBuffer> videoSample, FrameLayerInfo &frameLayerInfo) override; 71 Status GetFrameLayerInfo(uint32_t frameId, FrameLayerInfo &frameLayerInfo) override; 72 Status GetGopLayerInfo(uint32_t gopId, GopLayerInfo &gopLayerInfo) override; 73 Status GetIFramePos(std::vector<uint32_t> &IFramePos) override; 74 Status Dts2FrameId(int64_t dts, uint32_t &frameId, bool offset = true) override; 75 76 Status GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex, 77 const uint64_t relativePresentationTimeUs, uint32_t &index) override; 78 Status GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex, 79 const uint32_t index, uint64_t &relativePresentationTimeUs) override; 80 void SetCacheLimit(uint32_t limitSize) override; 81 82 private: 83 enum DumpMode : unsigned long { 84 DUMP_NONE = 0, 85 DUMP_READAT_INPUT = 0b001, 86 DUMP_AVPACKET_OUTPUT = 0b010, 87 DUMP_AVBUFFER_OUTPUT = 0b100, 88 }; 89 enum IndexAndPTSConvertMode : unsigned int { 90 GET_FIRST_PTS, 91 INDEX_TO_RELATIVEPTS, 92 RELATIVEPTS_TO_INDEX, 93 GET_ALL_FRAME_PTS, 94 }; 95 struct IOContext { 96 std::shared_ptr<DataSource> dataSource {nullptr}; 97 int64_t offset {0}; 98 uint64_t fileSize {0}; 99 bool eos {false}; 100 std::atomic<bool> retry {false}; 101 uint32_t initDownloadDataSize {0}; 102 std::atomic<bool> initCompleted {false}; 103 DumpMode dumpMode {DUMP_NONE}; 104 }; 105 void ConvertCsdToAnnexb(const AVStream& avStream, Meta &format); 106 int64_t GetFileDuration(const AVFormatContext& avFormatContext); 107 int64_t GetStreamDuration(const AVStream& avStream); 108 109 static int AVReadPacket(void* opaque, uint8_t* buf, int bufSize); 110 static int AVWritePacket(void* opaque, uint8_t* buf, int bufSize); 111 static int64_t AVSeek(void* opaque, int64_t offset, int whence); 112 AVIOContext* AllocAVIOContext(int flags, IOContext *ioContext); 113 std::shared_ptr<AVFormatContext> InitAVFormatContext(IOContext *ioContext); 114 static int CheckContextIsValid(void* opaque, int &bufSize); 115 void NotifyInitializationCompleted(); 116 117 void InitParser(); 118 void InitBitStreamContext(const AVStream& avStream); 119 Status ConvertAvcToAnnexb(AVPacket& pkt); 120 Status PushEOSToAllCache(); 121 bool TrackIsSelected(const uint32_t trackId); 122 Status ReadPacketToCacheQueue(const uint32_t readId); 123 Status AddPacketToCacheQueue(AVPacket *pkt); 124 Status SetDrmCencInfo(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 125 void WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 126 Status ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket); 127 Status ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* avpacket, 128 std::shared_ptr<SamplePacket> dstSamplePacket); 129 Status SetEosSample(std::shared_ptr<AVBuffer> sample); 130 Status WriteBuffer(std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize); 131 void ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize, 132 std::multimap<std::string, std::vector<uint8_t>>& drmInfo); 133 bool GetNextFrame(const uint8_t *data, const uint32_t size); 134 bool NeedCombineFrame(uint32_t trackId); 135 AVPacket* CombinePackets(std::shared_ptr<SamplePacket> samplePacket); 136 Status ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 137 Status ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket); 138 Status GetSeiInfo(); 139 140 int FindNaluSpliter(int size, const uint8_t *data); 141 bool CanDropAvcPkt(const AVPacket& pkt); 142 bool CanDropHevcPkt(const AVPacket& pkt); 143 bool WebvttPktProcess(AVPacket *pkt); 144 bool IsWebvttMP4(const AVStream *avStream); 145 void WebvttMP4EOSProcess(const AVPacket *pkt); 146 Status CheckCacheDataLimit(uint32_t trackId); 147 148 Status GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode, 149 uint32_t trackIndex, int64_t absolutePTS, uint32_t index); 150 Status PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode, 151 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 152 Status PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode, 153 const AVStream* avStream, int64_t absolutePTS, uint32_t index); 154 void InitPTSandIndexConvert(); 155 void IndexToRelativePTSProcess(int64_t pts, uint32_t index); 156 void RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS); 157 void PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode, 158 int64_t pts, int64_t absolutePTS, uint32_t index); 159 int64_t absolutePTSIndexZero_ = INT64_MAX; 160 std::priority_queue<int64_t> indexToRelativePTSMaxHeap_; 161 uint32_t indexToRelativePTSFrameCount_ = 0; 162 uint32_t relativePTSToIndexPosition_ = 0; 163 int64_t relativePTSToIndexPTSMin_ = INT64_MAX; 164 int64_t relativePTSToIndexPTSMax_ = INT64_MIN; 165 int64_t relativePTSToIndexRightDiff_ = INT64_MAX; 166 int64_t relativePTSToIndexLeftDiff_ = INT64_MAX; 167 int64_t relativePTSToIndexTempDiff_ = INT64_MAX; 168 void ParserFirstDts(); 169 Status InitIoContext(); 170 Status ParserRefInit(); 171 Status ParserRefInfoLoop(AVPacket *pkt, uint32_t curStreamId); 172 Status SelectProGopId(); 173 void ParserBoxInfo(); 174 175 std::mutex mutex_ {}; 176 std::shared_mutex sharedMutex_; 177 std::unordered_map<uint32_t, std::shared_ptr<std::mutex>> trackMtx_; 178 Seekable seekable_; 179 IOContext ioContext_; 180 std::vector<uint32_t> selectedTrackIds_; 181 BlockQueuePool cacheQueue_; 182 183 std::shared_ptr<AVInputFormat> pluginImpl_ {nullptr}; 184 std::shared_ptr<AVFormatContext> formatContext_ {nullptr}; 185 std::shared_ptr<AVBSFContext> avbsfContext_ {nullptr}; 186 std::shared_ptr<StreamParserManager> streamParser_ {nullptr}; 187 bool streamParserInited_ {false}; 188 189 Status GetVideoFirstKeyFrame(uint32_t trackIndex); 190 void ParseHEVCMetadataInfo(const AVStream& avStream, Meta &format); 191 AVPacket *firstFrame_ = nullptr; 192 193 std::atomic<bool> parserState_ = true; 194 IOContext parserRefIoContext_; 195 std::shared_ptr<AVFormatContext> parserRefFormatContext_{nullptr}; 196 int parserRefVideoStreamIdx_ = -1; 197 std::shared_ptr<ReferenceParserManager> referenceParser_{nullptr}; 198 int32_t parserCurGopId_ = 0; 199 int64_t pendingSeekMsTime_ = -1; 200 int64_t parserRefStartTime_ = -1; 201 std::list<uint32_t> processingIFrame_; 202 std::vector<uint32_t> IFramePos_; 203 double fps_{0}; 204 int64_t firstDts_ = 0; 205 uint32_t dtsOffset_ = 0; 206 bool isSdtpExist_ = false; 207 std::mutex syncMutex_; 208 bool updatePosIsForward_ = true; 209 bool isInit_ = false; 210 uint32_t cachelimitSize_ = 0; 211 bool outOfLimit_ = false; 212 bool setLimitByUser = false; 213 214 // dfx 215 struct TrackDfxInfo { 216 int frameIndex = 0; // for each track 217 int64_t lastPts; 218 int64_t lastPos; 219 int64_t lastDurantion; 220 }; 221 struct DumpParam { 222 DumpMode mode; 223 uint8_t* buf; 224 int trackId; 225 int64_t offset; 226 int size; 227 int index; 228 int64_t pts; 229 int64_t pos; 230 }; 231 std::unordered_map<int, TrackDfxInfo> trackDfxInfoMap_; 232 DumpMode dumpMode_ {DUMP_NONE}; 233 static std::atomic<int> readatIndex_; 234 int avpacketIndex_ {0}; 235 236 static void Dump(const DumpParam &dumpParam); 237 238 std::vector<int64_t> ptsListOrg_; 239 std::vector<int64_t> ptsListFromZero_; 240 std::unordered_map<int32_t, int64_t> iFramePtsMap_; 241 Status GetGopIdFromSeekPos(int64_t seekMs, int32_t &gopId); 242 Status ParserRefCheckVideoValid(const AVStream *videoStream); 243 }; 244 } // namespace Ffmpeg 245 } // namespace Plugins 246 } // namespace Media 247 } // namespace OHOS 248 #endif // FFMPEG_DEMUXER_PLUGIN_H 249