1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #define MEDIA_PLUGIN
17 #define HST_LOG_TAG "FfmpegDemuxerPlugin"
18 #include <unistd.h>
19 #include <algorithm>
20 #include <malloc.h>
21 #include <string>
22 #include <sstream>
23 #include <iomanip>
24 #include <map>
25 #include <fstream>
26 #include <chrono>
27 #include <limits>
28 #include "avcodec_trace.h"
29 #include "securec.h"
30 #include "ffmpeg_format_helper.h"
31 #include "ffmpeg_utils.h"
32 #include "buffer/avbuffer.h"
33 #include "plugin/plugin_buffer.h"
34 #include "plugin/plugin_definition.h"
35 #include "common/log.h"
36 #include "meta/video_types.h"
37 #include "avcodec_sysevent.h"
38 #include "demuxer_log_compressor.h"
39 #include "ffmpeg_demuxer_plugin.h"
40 #include "meta/format.h"
41 #include "syspara/parameters.h"
42
43 namespace {
44 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_DEMUXER, "FfmpegDemuxerPlugin" };
45 }
46
47 namespace OHOS {
48 namespace Media {
49 namespace Plugins {
50 namespace Ffmpeg {
51 const uint32_t DEFAULT_READ_SIZE = 4096;
52 const uint32_t DEFAULT_SNIFF_SIZE = 4096 * 4;
53 const int32_t MP3_PROBE_SCORE_LIMIT = 5;
54 const uint32_t STR_MAX_LEN = 4;
55 const uint32_t RANK_MAX = 100;
56 const uint32_t NAL_START_CODE_SIZE = 4;
57 const uint32_t INIT_DOWNLOADS_DATA_SIZE_THRESHOLD = 2 * 1024 * 1024;
58 const int64_t LIVE_FLV_PROBE_SIZE = 100 * 1024 * 2;
59 const uint32_t DEFAULT_CACHE_LIMIT = 50 * 1024 * 1024; // 50M
60 const int32_t INIT_TIME_THRESHOLD = 1000;
61 const uint32_t ID3V2_HEADER_SIZE = 10;
62
63 // id3v2 tag position
64 const int32_t POS_0 = 0;
65 const int32_t POS_1 = 1;
66 const int32_t POS_2 = 2;
67 const int32_t POS_3 = 3;
68 const int32_t POS_4 = 4;
69 const int32_t POS_5 = 4;
70 const int32_t POS_6 = 6;
71 const int32_t POS_7 = 7;
72 const int32_t POS_8 = 8;
73 const int32_t POS_9 = 9;
74 const int32_t POS_14 = 14;
75 const int32_t POS_21 = 21;
76 const int32_t POS_FF = 0xff;
77 const int32_t LEN_MASK = 0x7f;
78 const int32_t TAG_MASK = 0x80;
79 const int32_t TAG_VERSION_MASK = 0x10;
80
81 namespace {
82 std::map<std::string, std::shared_ptr<AVInputFormat>> g_pluginInputFormat;
83 std::mutex g_mtx;
84
85 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource);
86
87 Status RegisterPlugins(const std::shared_ptr<Register>& reg);
88
89 void ReplaceDelimiter(const std::string &delmiters, char newDelimiter, std::string &str);
90
91 static const std::map<SeekMode, int32_t> g_seekModeToFFmpegSeekFlags = {
92 { SeekMode::SEEK_PREVIOUS_SYNC, AVSEEK_FLAG_BACKWARD },
93 { SeekMode::SEEK_NEXT_SYNC, AVSEEK_FLAG_FRAME },
94 { SeekMode::SEEK_CLOSEST_SYNC, AVSEEK_FLAG_FRAME | AVSEEK_FLAG_BACKWARD }
95 };
96
97 static const std::map<AVCodecID, std::string> g_bitstreamFilterMap = {
98 { AV_CODEC_ID_H264, "h264_mp4toannexb" },
99 };
100
101 static const std::map<AVCodecID, StreamType> g_streamParserMap = {
102 { AV_CODEC_ID_HEVC, StreamType::HEVC },
103 { AV_CODEC_ID_VVC, StreamType::VVC },
104 };
105
106 static const std::vector<AVMediaType> g_streamMediaTypeVec = {
107 AVMEDIA_TYPE_AUDIO,
108 AVMEDIA_TYPE_VIDEO,
109 AVMEDIA_TYPE_SUBTITLE,
110 AVMEDIA_TYPE_TIMEDMETA
111 };
112
113 static std::vector<AVCodecID> g_imageCodecID = {
114 AV_CODEC_ID_MJPEG,
115 AV_CODEC_ID_PNG,
116 AV_CODEC_ID_PAM,
117 AV_CODEC_ID_BMP,
118 AV_CODEC_ID_JPEG2000,
119 AV_CODEC_ID_TARGA,
120 AV_CODEC_ID_TIFF,
121 AV_CODEC_ID_GIF,
122 AV_CODEC_ID_PCX,
123 AV_CODEC_ID_XWD,
124 AV_CODEC_ID_XBM,
125 AV_CODEC_ID_WEBP,
126 AV_CODEC_ID_APNG,
127 AV_CODEC_ID_XPM,
128 AV_CODEC_ID_SVG,
129 };
130
HaveValidParser(const AVCodecID codecId)131 bool HaveValidParser(const AVCodecID codecId)
132 {
133 return g_streamParserMap.count(codecId) != 0;
134 }
135
GetFileDuration(const AVFormatContext & avFormatContext)136 int64_t GetFileDuration(const AVFormatContext& avFormatContext)
137 {
138 int64_t duration = 0;
139 const AVDictionaryEntry *metaDuration = av_dict_get(avFormatContext.metadata, "DURATION", NULL, 0);
140 int64_t us;
141 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
142 if (us > duration) {
143 MEDIA_LOG_D("Get duration from file");
144 duration = us;
145 }
146 }
147
148 if (duration <= 0) {
149 for (uint32_t i = 0; i < avFormatContext.nb_streams; ++i) {
150 auto streamDuration = (ConvertTimeFromFFmpeg(avFormatContext.streams[i]->duration,
151 avFormatContext.streams[i]->time_base)) / 1000; // us
152 if (streamDuration > duration) {
153 MEDIA_LOG_D("Get duration from stream " PUBLIC_LOG_U32, i);
154 duration = streamDuration;
155 }
156 }
157 }
158 return duration;
159 }
160
GetStreamDuration(const AVStream & avStream)161 int64_t GetStreamDuration(const AVStream& avStream)
162 {
163 int64_t duration = 0;
164 const AVDictionaryEntry *metaDuration = av_dict_get(avStream.metadata, "DURATION", NULL, 0);
165 int64_t us;
166 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
167 if (us > duration) {
168 MEDIA_LOG_D("Get duration from stream");
169 duration = us;
170 }
171 }
172 return duration;
173 }
174
CheckStartTime(const AVFormatContext * formatContext,const AVStream * stream,int64_t & timeStamp,int64_t seekTime)175 bool CheckStartTime(const AVFormatContext *formatContext, const AVStream *stream, int64_t &timeStamp, int64_t seekTime)
176 {
177 int64_t startTime = 0;
178 int64_t num = 1000; // ms convert us
179 FALSE_RETURN_V_MSG_E(stream != nullptr, false, "String is nullptr");
180 if (stream->start_time != AV_NOPTS_VALUE) {
181 startTime = stream->start_time;
182 if (timeStamp > 0 && startTime > INT64_MAX - timeStamp) {
183 MEDIA_LOG_E("Seek value overflow with start time: " PUBLIC_LOG_D64 " timeStamp: " PUBLIC_LOG_D64,
184 startTime, timeStamp);
185 return false;
186 }
187 }
188 MEDIA_LOG_D("StartTime: " PUBLIC_LOG_D64, startTime);
189 int64_t fileDuration = formatContext->duration;
190 int64_t streamDuration = stream->duration;
191 if (fileDuration == AV_NOPTS_VALUE || fileDuration <= 0) {
192 fileDuration = GetFileDuration(*formatContext);
193 }
194 if (streamDuration == AV_NOPTS_VALUE || streamDuration <= 0) {
195 streamDuration = GetStreamDuration(*stream);
196 }
197 MEDIA_LOG_D("File duration=" PUBLIC_LOG_D64 ", stream duration=" PUBLIC_LOG_D64, fileDuration, streamDuration);
198 // when timestemp out of file duration, return error
199 if (fileDuration > 0 && seekTime * num > fileDuration) { // fileDuration us
200 MEDIA_LOG_E("Seek to timestamp=" PUBLIC_LOG_D64 " failed, max=" PUBLIC_LOG_D64, timeStamp, fileDuration);
201 return false;
202 }
203 // when timestemp out of stream duration, seek to end of stream
204 if (streamDuration > 0 && timeStamp > streamDuration) {
205 MEDIA_LOG_W("Out of stream, seek to " PUBLIC_LOG_D64, timeStamp);
206 timeStamp = streamDuration;
207 }
208 if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
209 MEDIA_LOG_D("Reset timeStamp by start time [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
210 timeStamp, timeStamp + startTime);
211 timeStamp += startTime;
212 }
213 return true;
214 }
215
ConvertFlagsToFFmpeg(AVStream * avStream,int64_t ffTime,SeekMode mode,int64_t seekTime)216 int ConvertFlagsToFFmpeg(AVStream *avStream, int64_t ffTime, SeekMode mode, int64_t seekTime)
217 {
218 FALSE_RETURN_V_MSG_E(avStream != nullptr && avStream->codecpar != nullptr, -1, "AVStream is nullptr");
219 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ffTime == 0) {
220 return AVSEEK_FLAG_FRAME;
221 }
222 if (avStream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO || seekTime == 0) {
223 return AVSEEK_FLAG_BACKWARD;
224 }
225 if (mode == SeekMode::SEEK_NEXT_SYNC || mode == SeekMode::SEEK_PREVIOUS_SYNC) {
226 return g_seekModeToFFmpegSeekFlags.at(mode);
227 }
228 // find closest time in next and prev
229 int keyFrameNext = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_FRAME);
230 FALSE_RETURN_V_MSG_E(keyFrameNext >= 0, AVSEEK_FLAG_BACKWARD, "Not next key frame");
231
232 int keyFramePrev = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_BACKWARD);
233 FALSE_RETURN_V_MSG_E(keyFramePrev >= 0, AVSEEK_FLAG_FRAME, "Not pre key frame");
234
235 int64_t ffTimePrev = CalculateTimeByFrameIndex(avStream, keyFramePrev);
236 int64_t ffTimeNext = CalculateTimeByFrameIndex(avStream, keyFrameNext);
237 MEDIA_LOG_D("FfTime=" PUBLIC_LOG_D64 ", ffTimePrev=" PUBLIC_LOG_D64 ", ffTimeNext=" PUBLIC_LOG_D64,
238 ffTime, ffTimePrev, ffTimeNext);
239 if (ffTimePrev == ffTimeNext || (ffTimeNext - ffTime < ffTime - ffTimePrev)) {
240 return AVSEEK_FLAG_FRAME;
241 } else {
242 return AVSEEK_FLAG_BACKWARD;
243 }
244 }
245
IsSupportedTrack(const AVStream & avStream)246 bool IsSupportedTrack(const AVStream& avStream)
247 {
248 FALSE_RETURN_V_MSG_E(avStream.codecpar != nullptr, false, "Codecpar is nullptr");
249 if (std::find(g_streamMediaTypeVec.cbegin(), g_streamMediaTypeVec.cend(),
250 avStream.codecpar->codec_type) == g_streamMediaTypeVec.cend()) {
251 MEDIA_LOG_E("Unsupport track type: " PUBLIC_LOG_S,
252 ConvertFFmpegMediaTypeToString(avStream.codecpar->codec_type).data());
253 return false;
254 }
255 if (avStream.codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
256 if (avStream.codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
257 MEDIA_LOG_E("Unsupport raw video track");
258 return false;
259 }
260 if (std::count(g_imageCodecID.begin(), g_imageCodecID.end(), avStream.codecpar->codec_id) > 0) {
261 MEDIA_LOG_E("Unsupport image track");
262 return false;
263 }
264 }
265 return true;
266 }
267 } // namespace
268
269 std::atomic<int> FFmpegDemuxerPlugin::readatIndex_ = 0;
FFmpegDemuxerPlugin(std::string name)270 FFmpegDemuxerPlugin::FFmpegDemuxerPlugin(std::string name)
271 : DemuxerPlugin(std::move(name)),
272 seekable_(Seekable::SEEKABLE),
273 ioContext_(),
274 selectedTrackIds_(),
275 cacheQueue_("cacheQueue"),
276 streamParserInited_(false),
277 parserRefIoContext_()
278 {
279 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
280 MEDIA_LOG_D("In");
281 #ifndef _WIN32
282 (void)mallopt(M_SET_THREAD_CACHE, M_THREAD_CACHE_DISABLE);
283 (void)mallopt(M_DELAYED_FREE, M_DELAYED_FREE_DISABLE);
284 #endif
285 av_log_set_callback(FfmpegLogPrint);
286 #ifdef BUILD_ENG_VERSION
287 std::string dumpModeStr = OHOS::system::GetParameter("FFmpegDemuxerPlugin.dump", "0");
288 dumpMode_ = static_cast<DumpMode>(strtoul(dumpModeStr.c_str(), nullptr, 2)); // 2 is binary
289 MEDIA_LOG_D("Dump mode = %s(%lu)", dumpModeStr.c_str(), dumpMode_);
290 #endif
291 MEDIA_LOG_D("Out");
292 }
293
~FFmpegDemuxerPlugin()294 FFmpegDemuxerPlugin::~FFmpegDemuxerPlugin()
295 {
296 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
297 MEDIA_LOG_D("In");
298 #ifndef _WIN32
299 (void)mallopt(M_FLUSH_THREAD_CACHE, 0);
300 #endif
301 formatContext_ = nullptr;
302 pluginImpl_ = nullptr;
303 avbsfContext_ = nullptr;
304 streamParser_ = nullptr;
305 referenceParser_ = nullptr;
306 parserRefFormatContext_ = nullptr;
307 selectedTrackIds_.clear();
308 if (firstFrame_ != nullptr) {
309 av_packet_free(&firstFrame_);
310 av_free(firstFrame_);
311 firstFrame_ = nullptr;
312 }
313 MEDIA_LOG_D("Out");
314 }
315
Dump(const DumpParam & dumpParam)316 void FFmpegDemuxerPlugin::Dump(const DumpParam &dumpParam)
317 {
318 std::string path;
319 switch (dumpParam.mode) {
320 case DUMP_READAT_INPUT:
321 path = "Readat_index." + std::to_string(dumpParam.index) + "_offset." + std::to_string(dumpParam.offset) +
322 "_size." + std::to_string(dumpParam.size);
323 break;
324 case DUMP_AVPACKET_OUTPUT:
325 path = "AVPacket_index." + std::to_string(dumpParam.index) + "_track." +
326 std::to_string(dumpParam.trackId) + "_pts." + std::to_string(dumpParam.pts) + "_pos." +
327 std::to_string(dumpParam.pos);
328 break;
329 case DUMP_AVBUFFER_OUTPUT:
330 path = "AVBuffer_track." + std::to_string(dumpParam.trackId) + "_index." +
331 std::to_string(dumpParam.index) + "_pts." + std::to_string(dumpParam.pts);
332 break;
333 default:
334 return;
335 }
336 std::ofstream ofs;
337 path = "/data/ff_dump/" + path;
338 ofs.open(path, std::ios::out); // | std::ios::app
339 if (ofs.is_open()) {
340 ofs.write(reinterpret_cast<char*>(dumpParam.buf), dumpParam.size);
341 ofs.close();
342 }
343 MEDIA_LOG_D("Dump path:" PUBLIC_LOG_S, path.c_str());
344 }
345
Reset()346 Status FFmpegDemuxerPlugin::Reset()
347 {
348 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
349 MEDIA_LOG_D("In");
350 readatIndex_ = 0;
351 avpacketIndex_ = 0;
352 ioContext_.offset = 0;
353 ioContext_.eos = false;
354 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
355 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
356 }
357 selectedTrackIds_.clear();
358 pluginImpl_.reset();
359 formatContext_.reset();
360 avbsfContext_.reset();
361 trackMtx_.clear();
362 trackDfxInfoMap_.clear();
363 return Status::OK;
364 }
365
InitBitStreamContext(const AVStream & avStream)366 void FFmpegDemuxerPlugin::InitBitStreamContext(const AVStream& avStream)
367 {
368 FALSE_RETURN_MSG(avStream.codecpar != nullptr, "Codecpar is nullptr");
369 AVCodecID codecID = avStream.codecpar->codec_id;
370 MEDIA_LOG_D("For track " PUBLIC_LOG_D32, avStream.index);
371 FALSE_RETURN_MSG(g_bitstreamFilterMap.count(codecID) != 0, "Can not match any BitStreamContext");
372 const AVBitStreamFilter* avBitStreamFilter = av_bsf_get_by_name(g_bitstreamFilterMap.at(codecID).c_str());
373
374 FALSE_RETURN_MSG((avBitStreamFilter != nullptr), "Call av_bsf_get_by_name failed for" PUBLIC_LOG_S,
375 g_bitstreamFilterMap.at(codecID).c_str());
376
377 if (!avbsfContext_) {
378 AVBSFContext* avbsfContext {nullptr};
379 int ret = av_bsf_alloc(avBitStreamFilter, &avbsfContext);
380 FALSE_RETURN_MSG((ret >= 0 && avbsfContext != nullptr),
381 "Call av_bsf_alloc failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
382
383 ret = avcodec_parameters_copy(avbsfContext->par_in, avStream.codecpar);
384 FALSE_RETURN_MSG((ret >= 0), "Call avcodec_parameters_copy failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
385
386 ret = av_bsf_init(avbsfContext);
387 FALSE_RETURN_MSG((ret >= 0), "Call av_bsf_init failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
388
389 avbsfContext_ = std::shared_ptr<AVBSFContext>(avbsfContext, [](AVBSFContext* ptr) {
390 if (ptr) {
391 av_bsf_free(&ptr);
392 }
393 });
394 }
395 FALSE_RETURN_MSG(avbsfContext_ != nullptr, "Stream " PUBLIC_LOG_S " will not be converted to annexb",
396 g_bitstreamFilterMap.at(codecID).c_str());
397 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will convert to annexb", avStream.index);
398 }
399
ConvertAvcToAnnexb(AVPacket & pkt)400 Status FFmpegDemuxerPlugin::ConvertAvcToAnnexb(AVPacket& pkt)
401 {
402 int ret = av_bsf_send_packet(avbsfContext_.get(), &pkt);
403 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
404 "Call av_bsf_send_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
405 av_packet_unref(&pkt);
406 ret = av_bsf_receive_packet(avbsfContext_.get(), &pkt);
407 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
408 "Call av_bsf_receive_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
409 return Status::OK;
410 }
411
ConvertHevcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)412 Status FFmpegDemuxerPlugin::ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
413 {
414 size_t cencInfoSize = 0;
415 uint8_t *cencInfo = av_packet_get_side_data(samplePacket->pkts[0], AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
416 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, cencInfo, cencInfoSize, false);
417 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) &&
418 streamParser_->IsSyncFrame(pkt.data, pkt.size)) {
419 pkt.flags = static_cast<int32_t>(static_cast<uint32_t>(pkt.flags) | static_cast<uint32_t>(AV_PKT_FLAG_KEY));
420 }
421 return Status::OK;
422 }
423
ConvertVvcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)424 Status FFmpegDemuxerPlugin::ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
425 {
426 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, nullptr, 0, false);
427 return Status::OK;
428 }
429
WriteBuffer(std::shared_ptr<AVBuffer> outBuffer,const uint8_t * writeData,int32_t writeSize)430 Status FFmpegDemuxerPlugin::WriteBuffer(
431 std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize)
432 {
433 FALSE_RETURN_V_MSG_E(outBuffer != nullptr, Status::ERROR_NULL_POINTER, "Buffer is nullptr");
434 if (writeData != nullptr && writeSize > 0) {
435 FALSE_RETURN_V_MSG_E(outBuffer->memory_ != nullptr, Status::ERROR_NULL_POINTER, "Memory is nullptr");
436 int32_t ret = outBuffer->memory_->Write(writeData, writeSize, 0);
437 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_INVALID_OPERATION, "Memory write failed");
438 }
439
440 MEDIA_LOG_D("CurrentBuffer: [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_U32 "]",
441 outBuffer->pts_, outBuffer->duration_, outBuffer->flag_);
442 return Status::OK;
443 }
444
SetDrmCencInfo(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)445 Status FFmpegDemuxerPlugin::SetDrmCencInfo(
446 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
447 {
448 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
449 "Sample is nullptr");
450 FALSE_RETURN_V_MSG_E((samplePacket != nullptr && samplePacket->pkts.size() > 0), Status::ERROR_INVALID_OPERATION,
451 "Packet is nullptr");
452 FALSE_RETURN_V_MSG_E((samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0),
453 Status::ERROR_INVALID_OPERATION, "Packet empty");
454
455 size_t cencInfoSize = 0;
456 MetaDrmCencInfo *cencInfo = (MetaDrmCencInfo *)av_packet_get_side_data(samplePacket->pkts[0],
457 AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
458 if ((cencInfo != nullptr) && (cencInfoSize != 0)) {
459 std::vector<uint8_t> drmCencVec(reinterpret_cast<uint8_t *>(cencInfo),
460 (reinterpret_cast<uint8_t *>(cencInfo)) + sizeof(MetaDrmCencInfo));
461 sample->meta_->SetData(Media::Tag::DRM_CENC_INFO, std::move(drmCencVec));
462 }
463 return Status::OK;
464 }
465
GetNextFrame(const uint8_t * data,const uint32_t size)466 bool FFmpegDemuxerPlugin::GetNextFrame(const uint8_t *data, const uint32_t size)
467 {
468 if (size < NAL_START_CODE_SIZE) {
469 return false;
470 }
471 bool hasShortStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 1); // 001
472 bool hasLongStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1); // 0001
473 return hasShortStartCode || hasLongStartCode;
474 }
475
NeedCombineFrame(uint32_t trackId)476 bool FFmpegDemuxerPlugin::NeedCombineFrame(uint32_t trackId)
477 {
478 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, false, "AVFormatContext is nullptr");
479 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MPEGTS &&
480 formatContext_->streams[trackId]->codecpar->codec_id == AV_CODEC_ID_HEVC) {
481 return true;
482 }
483 return false;
484 }
485
CombinePackets(std::shared_ptr<SamplePacket> samplePacket)486 AVPacket* FFmpegDemuxerPlugin::CombinePackets(std::shared_ptr<SamplePacket> samplePacket)
487 {
488 AVPacket *tempPkt = nullptr;
489 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) && samplePacket->pkts.size() > 1) {
490 int totalSize = 0;
491 for (auto pkt : samplePacket->pkts) {
492 FALSE_RETURN_V_MSG_E(pkt != nullptr, nullptr, "AVPacket is nullptr");
493 totalSize += pkt->size;
494 }
495 tempPkt = av_packet_alloc();
496 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, nullptr, "Temp packet is nullptr");
497 int ret = av_new_packet(tempPkt, totalSize);
498 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "Call av_new_packet failed");
499 av_packet_copy_props(tempPkt, samplePacket->pkts[0]);
500 int offset = 0;
501 bool copySuccess = true;
502 for (auto pkt : samplePacket->pkts) {
503 if (pkt == nullptr) {
504 copySuccess = false;
505 MEDIA_LOG_E("Cache packet is nullptr");
506 break;
507 }
508 ret = memcpy_s(tempPkt->data + offset, pkt->size, pkt->data, pkt->size);
509 if (ret != EOK) {
510 copySuccess = false;
511 MEDIA_LOG_E("Memcpy failed, ret:" PUBLIC_LOG_D32, ret);
512 break;
513 }
514 offset += pkt->size;
515 }
516 if (!copySuccess) {
517 av_packet_free(&tempPkt);
518 av_free(tempPkt);
519 tempPkt = nullptr;
520 return nullptr;
521 }
522 tempPkt->size = totalSize;
523 MEDIA_LOG_D("Combine " PUBLIC_LOG_ZU " packets, total size=" PUBLIC_LOG_D32,
524 samplePacket->pkts.size(), totalSize);
525 } else {
526 tempPkt = samplePacket->pkts[0];
527 }
528 return tempPkt;
529 }
530
ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample,AVPacket * srcAVPacket,std::shared_ptr<SamplePacket> dstSamplePacket)531 Status FFmpegDemuxerPlugin::ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* srcAVPacket,
532 std::shared_ptr<SamplePacket> dstSamplePacket)
533 {
534 Status ret = Status::OK;
535 auto codecId = formatContext_->streams[srcAVPacket->stream_index]->codecpar->codec_id;
536 if (codecId == AV_CODEC_ID_HEVC && streamParser_ != nullptr && streamParserInited_) {
537 ret = ConvertHevcToAnnexb(*srcAVPacket, dstSamplePacket);
538 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_HEVC);
539 } else if (codecId == AV_CODEC_ID_VVC && streamParser_ != nullptr && streamParserInited_) {
540 ret = ConvertVvcToAnnexb(*srcAVPacket, dstSamplePacket);
541 } else if (codecId == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
542 ret = ConvertAvcToAnnexb(*srcAVPacket);
543 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_H264);
544 }
545 return ret;
546 }
547
WriteBufferAttr(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)548 void FFmpegDemuxerPlugin::WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
549 {
550 AVStream *avStream = formatContext_->streams[samplePacket->pkts[0]->stream_index];
551 if (samplePacket->pkts[0]->pts != AV_NOPTS_VALUE) {
552 sample->pts_ = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->pts, avStream->time_base));
553 }
554 // durantion dts
555 if (samplePacket->pkts[0]->duration != AV_NOPTS_VALUE) {
556 int64_t duration = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->duration, avStream->time_base));
557 sample->duration_ = duration;
558 sample->meta_->SetData(Media::Tag::BUFFER_DURATION, duration);
559 }
560 if (samplePacket->pkts[0]->dts != AV_NOPTS_VALUE) {
561 int64_t dts = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->dts, avStream->time_base));
562 sample->dts_ = dts;
563 sample->meta_->SetData(Media::Tag::BUFFER_DECODING_TIMESTAMP, dts);
564 }
565 }
566
ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)567 Status FFmpegDemuxerPlugin::ConvertAVPacketToSample(
568 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
569 {
570 FALSE_RETURN_V_MSG_E(samplePacket != nullptr && samplePacket->pkts.size() > 0 &&
571 samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0,
572 Status::ERROR_INVALID_OPERATION, "Input packet is nullptr or empty");
573 MEDIA_LOG_D("Convert packet info for track " PUBLIC_LOG_D32, samplePacket->pkts[0]->stream_index);
574 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
575 "Input sample is nullptr");
576
577 WriteBufferAttr(sample, samplePacket);
578
579 // convert
580 AVPacket *tempPkt = CombinePackets(samplePacket);
581 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, Status::ERROR_INVALID_OPERATION, "Temp packet is empty");
582 Status ret = ConvertPacketToAnnexb(sample, tempPkt, samplePacket);
583 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Convert annexb failed");
584
585 // flag\copy
586 int32_t remainSize = tempPkt->size - static_cast<int32_t>(samplePacket->offset);
587 int32_t copySize = remainSize < sample->memory_->GetCapacity() ? remainSize : sample->memory_->GetCapacity();
588 MEDIA_LOG_D("Convert size [" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
589 tempPkt->size, remainSize, copySize, samplePacket->offset);
590 uint32_t flag = ConvertFlagsFromFFmpeg(*tempPkt, (copySize != tempPkt->size));
591 SetDrmCencInfo(sample, samplePacket);
592
593 sample->flag_ = flag;
594 ret = WriteBuffer(sample, tempPkt->data + samplePacket->offset, copySize);
595 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
596
597 if (!samplePacket->isEOS) {
598 trackDfxInfoMap_[tempPkt->stream_index].lastPts = sample->pts_;
599 trackDfxInfoMap_[tempPkt->stream_index].lastDurantion = sample->duration_;
600 trackDfxInfoMap_[tempPkt->stream_index].lastPos = tempPkt->pos;
601 }
602 #ifdef BUILD_ENG_VERSION
603 DumpParam dumpParam {DumpMode(DUMP_AVBUFFER_OUTPUT & dumpMode_), tempPkt->data + samplePacket->offset,
604 tempPkt->stream_index, -1, copySize, trackDfxInfoMap_[tempPkt->stream_index].frameIndex++, tempPkt->pts, -1};
605 Dump(dumpParam);
606 #endif
607 if (tempPkt != nullptr && tempPkt->size != samplePacket->pkts[0]->size) {
608 av_packet_free(&tempPkt);
609 av_free(tempPkt);
610 tempPkt = nullptr;
611 }
612
613 if (copySize < remainSize) {
614 samplePacket->offset += static_cast<uint32_t>(copySize);
615 MEDIA_LOG_D("Buffer is not enough, next buffer to copy remain data");
616 return Status::ERROR_NOT_ENOUGH_DATA;
617 }
618 return Status::OK;
619 }
620
PushEOSToAllCache()621 Status FFmpegDemuxerPlugin::PushEOSToAllCache()
622 {
623 Status ret = Status::OK;
624 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
625 auto streamIndex = selectedTrackIds_[i];
626 MEDIA_LOG_I("Track " PUBLIC_LOG_D32, streamIndex);
627 std::shared_ptr<SamplePacket> eosSample = std::make_shared<SamplePacket>();
628 eosSample->isEOS = true;
629 cacheQueue_.Push(streamIndex, eosSample);
630 ret = CheckCacheDataLimit(streamIndex);
631 }
632 return ret;
633 }
634
WebvttPktProcess(AVPacket * pkt)635 bool FFmpegDemuxerPlugin::WebvttPktProcess(AVPacket *pkt)
636 {
637 auto trackId = pkt->stream_index;
638 if (pkt->size > 0) { // vttc
639 return false;
640 } else { // vtte
641 if (cacheQueue_.HasCache(trackId)) {
642 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
643 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
644 cacheSamplePacket->pkts[0]->duration = pkt->pts - cacheSamplePacket->pkts[0]->pts;
645 }
646 }
647 }
648 av_packet_free(&pkt);
649 return true;
650 }
651
IsWebvttMP4(const AVStream * avStream)652 bool FFmpegDemuxerPlugin::IsWebvttMP4(const AVStream *avStream)
653 {
654 if (avStream->codecpar->codec_id == AV_CODEC_ID_WEBVTT &&
655 FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4) {
656 return true;
657 }
658 return false;
659 }
660
WebvttMP4EOSProcess(const AVPacket * pkt)661 void FFmpegDemuxerPlugin::WebvttMP4EOSProcess(const AVPacket *pkt)
662 {
663 if (pkt != nullptr) {
664 auto trackId = pkt->stream_index;
665 AVStream *avStream = formatContext_->streams[trackId];
666 if (IsWebvttMP4(avStream) && pkt->size == 0 && cacheQueue_.HasCache(trackId)) {
667 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
668 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
669 cacheSamplePacket->pkts[0]->duration =
670 formatContext_->streams[pkt->stream_index]->duration - cacheSamplePacket->pkts[0]->pts;
671 }
672 }
673 }
674 }
675
ReadPacketToCacheQueue(const uint32_t readId)676 Status FFmpegDemuxerPlugin::ReadPacketToCacheQueue(const uint32_t readId)
677 {
678 std::lock_guard<std::mutex> lock(mutex_);
679 AVPacket *pkt = nullptr;
680 bool continueRead = true;
681 Status ret = Status::OK;
682 while (continueRead) {
683 if (pkt == nullptr) {
684 pkt = av_packet_alloc();
685 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
686 }
687 std::unique_lock<std::mutex> sLock(syncMutex_);
688 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
689 sLock.unlock();
690 if (ffmpegRet == AVERROR_EOF) { // eos
691 WebvttMP4EOSProcess(pkt);
692 av_packet_free(&pkt);
693 ret = PushEOSToAllCache();
694 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Push eos failed");
695 return Status::END_OF_STREAM;
696 }
697 if (ffmpegRet < 0) { // fail
698 av_packet_free(&pkt);
699 MEDIA_LOG_E("Call av_read_frame failed:" PUBLIC_LOG_S ", retry: " PUBLIC_LOG_D32,
700 AVStrError(ffmpegRet).c_str(), int(ioContext_.retry));
701 if (ioContext_.retry) {
702 formatContext_->pb->eof_reached = 0;
703 formatContext_->pb->error = 0;
704 ioContext_.retry = false;
705 return Status::ERROR_AGAIN;
706 }
707 return Status::ERROR_UNKNOWN;
708 }
709 auto trackId = pkt->stream_index;
710 if (!TrackIsSelected(trackId)) {
711 av_packet_unref(pkt);
712 continue;
713 }
714 AVStream *avStream = formatContext_->streams[trackId];
715 if (IsWebvttMP4(avStream) && WebvttPktProcess(pkt)) {
716 break;
717 } else if (!IsWebvttMP4(avStream) && (!NeedCombineFrame(readId) ||
718 (cacheQueue_.HasCache(static_cast<uint32_t>(trackId)) && GetNextFrame(pkt->data, pkt->size)))) {
719 continueRead = false;
720 }
721 ret = AddPacketToCacheQueue(pkt);
722 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Add cache failed");
723 pkt = nullptr;
724 }
725 return ret;
726 }
727
SetEosSample(std::shared_ptr<AVBuffer> sample)728 Status FFmpegDemuxerPlugin::SetEosSample(std::shared_ptr<AVBuffer> sample)
729 {
730 MEDIA_LOG_D("In");
731 sample->pts_ = 0;
732 sample->flag_ = (uint32_t)(AVBufferFlag::EOS);
733 Status ret = WriteBuffer(sample, nullptr, 0);
734 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
735 MEDIA_LOG_I("Out");
736 return Status::OK;
737 }
738
Start()739 Status FFmpegDemuxerPlugin::Start()
740 {
741 return Status::OK;
742 }
743
Stop()744 Status FFmpegDemuxerPlugin::Stop()
745 {
746 return Status::OK;
747 }
748
749 // Write packet unimplemented, return 0
AVWritePacket(void * opaque,uint8_t * buf,int bufSize)750 int FFmpegDemuxerPlugin::AVWritePacket(void* opaque, uint8_t* buf, int bufSize)
751 {
752 (void)opaque;
753 (void)buf;
754 (void)bufSize;
755 return 0;
756 }
757
CheckContextIsValid(void * opaque,int & bufSize)758 int FFmpegDemuxerPlugin::CheckContextIsValid(void* opaque, int &bufSize)
759 {
760 int ret = -1;
761 auto ioContext = static_cast<IOContext*>(opaque);
762 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
763 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, ret, "DataSource is nullptr");
764 FALSE_RETURN_V_MSG_E(ioContext->offset <= INT64_MAX - static_cast<int64_t>(bufSize), ret, "Offset invalid");
765
766 if (ioContext->dataSource->IsDash() && ioContext->eos == true) {
767 MEDIA_LOG_I("Read eos");
768 return AVERROR_EOF;
769 }
770
771 MEDIA_LOG_D("Offset: " PUBLIC_LOG_D64 ", totalSize: " PUBLIC_LOG_U64, ioContext->offset, ioContext->fileSize);
772 if (ioContext->fileSize > 0) {
773 FALSE_RETURN_V_MSG_E(static_cast<uint64_t>(ioContext->offset) <= ioContext->fileSize, ret, "Out of file size");
774 if (static_cast<size_t>(ioContext->offset + bufSize) > ioContext->fileSize) {
775 bufSize = static_cast<int64_t>(ioContext->fileSize) - ioContext->offset;
776 }
777 }
778 return 0;
779 }
780
781 // Write packet data into the buffer provided by ffmpeg
AVReadPacket(void * opaque,uint8_t * buf,int bufSize)782 int FFmpegDemuxerPlugin::AVReadPacket(void* opaque, uint8_t* buf, int bufSize)
783 {
784 int ret = CheckContextIsValid(opaque, bufSize);
785 FALSE_RETURN_V(ret == 0, ret);
786
787 ret = -1;
788 auto ioContext = static_cast<IOContext*>(opaque);
789 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
790 auto buffer = std::make_shared<Buffer>();
791 FALSE_RETURN_V_MSG_E(buffer != nullptr, ret, "Buffer is nullptr");
792 auto bufData = buffer->WrapMemory(buf, bufSize, 0);
793 FALSE_RETURN_V_MSG_E(buffer->GetMemory() != nullptr, ret, "Memory is nullptr");
794
795 MediaAVCodec::AVCodecTrace trace("AVReadPacket_ReadAt");
796 auto result = ioContext->dataSource->ReadAt(ioContext->offset, buffer, static_cast<size_t>(bufSize));
797 int dataSize = static_cast<int>(buffer->GetMemory()->GetSize());
798 MEDIA_LOG_D("Want:" PUBLIC_LOG_D32 ", Get:" PUBLIC_LOG_D32 ", offset:" PUBLIC_LOG_D64 ", index:" PUBLIC_LOG_D32,
799 bufSize, dataSize, ioContext->offset, readatIndex_.load());
800 #ifdef BUILD_ENG_VERSION
801 DumpParam dumpParam {DumpMode(DUMP_READAT_INPUT & ioContext->dumpMode), buf, -1, ioContext->offset,
802 dataSize, readatIndex_++, -1, -1};
803 Dump(dumpParam);
804 #endif
805 switch (result) {
806 case Status::OK:
807 case Status::ERROR_AGAIN:
808 if (dataSize == 0) {
809 MEDIA_LOG_I("Read again");
810 ioContext->retry = true;
811 } else {
812 ioContext->offset += dataSize;
813 ret = dataSize;
814 }
815 break;
816 case Status::END_OF_STREAM:
817 MEDIA_LOG_I("Read end");
818 ioContext->eos = true;
819 ret = AVERROR_EOF;
820 break;
821 default:
822 MEDIA_LOG_I("Read failed " PUBLIC_LOG_D32, static_cast<int>(result));
823 break;
824 }
825
826 if (!ioContext->initCompleted) {
827 if (ioContext->initDownloadDataSize <= UINT32_MAX - static_cast<uint32_t>(dataSize)) {
828 ioContext->initDownloadDataSize += static_cast<uint32_t>(dataSize);
829 } else {
830 MEDIA_LOG_W("DataSize " PUBLIC_LOG_U32 " is invalid", static_cast<uint32_t>(dataSize));
831 }
832 }
833
834 return ret;
835 }
836
AVSeek(void * opaque,int64_t offset,int whence)837 int64_t FFmpegDemuxerPlugin::AVSeek(void* opaque, int64_t offset, int whence)
838 {
839 auto ioContext = static_cast<IOContext*>(opaque);
840 uint64_t newPos = 0;
841 FALSE_RETURN_V_MSG_E(ioContext != nullptr, newPos, "IOContext is nullptr");
842 switch (whence) {
843 case SEEK_SET:
844 newPos = static_cast<uint64_t>(offset);
845 ioContext->offset = newPos;
846 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
847 whence, offset, newPos);
848 break;
849 case SEEK_CUR:
850 newPos = ioContext->offset + offset;
851 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
852 whence, offset, newPos);
853 break;
854 case SEEK_END:
855 case AVSEEK_SIZE: {
856 if (ioContext->dataSource->IsDash()) {
857 return -1;
858 }
859 uint64_t mediaDataSize = 0;
860 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, newPos, "DataSource is nullptr");
861 if (ioContext->dataSource->GetSize(mediaDataSize) == Status::OK && (mediaDataSize > 0)) {
862 newPos = mediaDataSize + offset;
863 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
864 whence, offset, newPos);
865 }
866 break;
867 }
868 default:
869 MEDIA_LOG_E("Unexpected whence " PUBLIC_LOG_D32, whence);
870 break;
871 }
872 if (whence != AVSEEK_SIZE) {
873 ioContext->offset = newPos;
874 }
875 MEDIA_LOG_D("Current offset: " PUBLIC_LOG_D64 ", new pos: " PUBLIC_LOG_U64, ioContext->offset, newPos);
876 return newPos;
877 }
878
AllocAVIOContext(int flags,IOContext * ioContext)879 AVIOContext* FFmpegDemuxerPlugin::AllocAVIOContext(int flags, IOContext *ioContext)
880 {
881 auto buffer = static_cast<unsigned char*>(av_malloc(DEFAULT_READ_SIZE));
882 FALSE_RETURN_V_MSG_E(buffer != nullptr, nullptr, "Call av_malloc failed");
883
884 AVIOContext* avioContext = avio_alloc_context(
885 buffer, DEFAULT_READ_SIZE, flags & AVIO_FLAG_WRITE, static_cast<void*>(ioContext),
886 AVReadPacket, AVWritePacket, AVSeek);
887 if (avioContext == nullptr) {
888 MEDIA_LOG_E("Call avio_alloc_context failed");
889 av_free(buffer);
890 return nullptr;
891 }
892 avioContext->seekable = (seekable_ == Seekable::SEEKABLE) ? AVIO_SEEKABLE_NORMAL : 0;
893 if (!(static_cast<uint32_t>(flags) & static_cast<uint32_t>(AVIO_FLAG_WRITE))) {
894 avioContext->buf_ptr = avioContext->buf_end;
895 avioContext->write_flag = 0;
896 }
897 return avioContext;
898 }
899
FreeContext(AVFormatContext * formatContext,AVIOContext * avioContext)900 void FreeContext(AVFormatContext* formatContext, AVIOContext* avioContext)
901 {
902 if (formatContext) {
903 avformat_close_input(&formatContext);
904 }
905 if (avioContext) {
906 if (avioContext->buffer) {
907 av_freep(&(avioContext->buffer));
908 }
909 avio_context_free(&avioContext);
910 }
911 }
912
ParseHeader(AVFormatContext * formatContext,std::shared_ptr<AVInputFormat> pluginImpl,AVDictionary ** options)913 int32_t ParseHeader(AVFormatContext* formatContext, std::shared_ptr<AVInputFormat> pluginImpl, AVDictionary **options)
914 {
915 FALSE_RETURN_V_MSG_E(formatContext && pluginImpl, -1, "AVFormatContext is nullptr");
916 MediaAVCodec::AVCodecTrace trace("ffmpeg_init");
917
918 AVIOContext* avioContext = formatContext->pb;
919 auto begin = std::chrono::system_clock::now();
920 int ret = avformat_open_input(&formatContext, nullptr, pluginImpl.get(), options);
921 if (ret < 0) {
922 FreeContext(formatContext, avioContext);
923 MEDIA_LOG_E("Call avformat_open_input failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
924 pluginImpl->name, AVStrError(ret).c_str());
925 return ret;
926 }
927
928 auto open = std::chrono::system_clock::now();
929 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext) == FileType::FLV) { // Fix init live-flv-source too slow
930 formatContext->probesize = LIVE_FLV_PROBE_SIZE;
931 }
932
933 ret = avformat_find_stream_info(formatContext, NULL);
934 auto parse = std::chrono::system_clock::now();
935 int openSpend = static_cast<int>(static_cast<std::chrono::duration<double, std::milli>>(open - begin).count());
936 int parseSpend = static_cast<int>(static_cast<std::chrono::duration<double, std::milli>>(parse - open).count());
937 if (openSpend + parseSpend > INIT_TIME_THRESHOLD) {
938 MEDIA_LOG_W("Spend [" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]", openSpend, parseSpend);
939 }
940 if (ret < 0) {
941 FreeContext(formatContext, avioContext);
942 MEDIA_LOG_E("Parse stream info failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
943 pluginImpl->name, AVStrError(ret).c_str());
944 return ret;
945 }
946 return 0;
947 }
948
InitAVFormatContext(IOContext * ioContext)949 std::shared_ptr<AVFormatContext> FFmpegDemuxerPlugin::InitAVFormatContext(IOContext *ioContext)
950 {
951 AVFormatContext* formatContext = avformat_alloc_context();
952 FALSE_RETURN_V_MSG_E(formatContext != nullptr, nullptr, "AVFormatContext is nullptr");
953
954 formatContext->pb = AllocAVIOContext(AVIO_FLAG_READ, ioContext);
955 if (formatContext->pb == nullptr) {
956 FreeContext(formatContext, nullptr);
957 return nullptr;
958 }
959
960 formatContext->flags = static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_CUSTOM_IO);
961 if (std::string(pluginImpl_->name) == "mp3") {
962 formatContext->flags =
963 static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_FAST_SEEK);
964 }
965 AVDictionary *options = nullptr;
966 if (ioContext_.dataSource->IsDash()) {
967 av_dict_set(&options, "use_tfdt", "true", 0);
968 }
969
970 int ret = ParseHeader(formatContext, pluginImpl_, &options);
971 av_dict_free(&options);
972 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "ParseHeader failed");
973
974 std::shared_ptr<AVFormatContext> retFormatContext =
975 std::shared_ptr<AVFormatContext>(formatContext, [](AVFormatContext *ptr) {
976 if (ptr) {
977 auto ctx = ptr->pb;
978 avformat_close_input(&ptr);
979 if (ctx) {
980 ctx->opaque = nullptr;
981 av_freep(&(ctx->buffer));
982 av_opt_free(ctx);
983 avio_context_free(&ctx);
984 ctx = nullptr;
985 }
986 }
987 });
988 return retFormatContext;
989 }
990
NotifyInitializationCompleted()991 void FFmpegDemuxerPlugin::NotifyInitializationCompleted()
992 {
993 ioContext_.initCompleted = true;
994 if (ioContext_.initDownloadDataSize >= INIT_DOWNLOADS_DATA_SIZE_THRESHOLD) {
995 MEDIA_LOG_I("Large init size %{public}u", ioContext_.initDownloadDataSize);
996 MediaAVCodec::DemuxerInitEventWrite(ioContext_.initDownloadDataSize, pluginName_);
997 }
998 }
999
SetDataSource(const std::shared_ptr<DataSource> & source)1000 Status FFmpegDemuxerPlugin::SetDataSource(const std::shared_ptr<DataSource>& source)
1001 {
1002 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1003 FALSE_RETURN_V_MSG_E(formatContext_ == nullptr, Status::ERROR_WRONG_STATE, "AVFormatContext is nullptr");
1004 FALSE_RETURN_V_MSG_E(source != nullptr, Status::ERROR_INVALID_PARAMETER, "DataSource is nullptr");
1005 ioContext_.dataSource = source;
1006 ioContext_.offset = 0;
1007 ioContext_.eos = false;
1008 ioContext_.dumpMode = dumpMode_;
1009 seekable_ = ioContext_.dataSource->IsDash() ? Plugins::Seekable::UNSEEKABLE : source->GetSeekable();
1010 if (seekable_ == Plugins::Seekable::SEEKABLE) {
1011 ioContext_.dataSource->GetSize(ioContext_.fileSize);
1012 } else {
1013 ioContext_.fileSize = -1;
1014 }
1015 MEDIA_LOG_I("FileSize: " PUBLIC_LOG_U64 ", seekable: " PUBLIC_LOG_D32, ioContext_.fileSize, seekable_);
1016 {
1017 std::lock_guard<std::mutex> glock(g_mtx);
1018 pluginImpl_ = g_pluginInputFormat[pluginName_];
1019 }
1020 FALSE_RETURN_V_MSG_E(pluginImpl_ != nullptr, Status::ERROR_UNSUPPORTED_FORMAT, "No match inputformat");
1021 formatContext_ = InitAVFormatContext(&ioContext_);
1022 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1023 InitParser();
1024
1025 NotifyInitializationCompleted();
1026 MEDIA_LOG_I("Out");
1027 cachelimitSize_ = DEFAULT_CACHE_LIMIT;
1028 return Status::OK;
1029 }
1030
InitParser()1031 void FFmpegDemuxerPlugin::InitParser()
1032 {
1033 FALSE_RETURN_MSG(formatContext_ != nullptr, "AVFormatContext is nullptr");
1034 ParserBoxInfo();
1035 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1036 if (g_bitstreamFilterMap.count(formatContext_->streams[trackIndex]->codecpar->codec_id) != 0) {
1037 InitBitStreamContext(*(formatContext_->streams[trackIndex]));
1038 break;
1039 }
1040 if (HaveValidParser(formatContext_->streams[trackIndex]->codecpar->codec_id)) {
1041 streamParser_ = StreamParserManager::Create(g_streamParserMap.at(
1042 formatContext_->streams[trackIndex]->codecpar->codec_id));
1043 if (streamParser_ == nullptr) {
1044 MEDIA_LOG_W("Init failed");
1045 } else {
1046 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will be converted", trackIndex);
1047 }
1048 break;
1049 }
1050 }
1051 }
1052
GetSeiInfo()1053 Status FFmpegDemuxerPlugin::GetSeiInfo()
1054 {
1055 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1056 Status ret = Status::OK;
1057 if (streamParser_ != nullptr && !streamParserInited_) {
1058 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1059 auto avStream = formatContext_->streams[trackIndex];
1060 if (HaveValidParser(avStream->codecpar->codec_id)) {
1061 ret = GetVideoFirstKeyFrame(trackIndex);
1062 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, Status::ERROR_NO_MEMORY, "No memory");
1063 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr && firstFrame_->data != nullptr,
1064 Status::ERROR_WRONG_STATE, "Get first frame failed");
1065 streamParser_->ConvertExtraDataToAnnexb(
1066 avStream->codecpar->extradata, avStream->codecpar->extradata_size);
1067 streamParserInited_ = true;
1068 break;
1069 }
1070 }
1071 }
1072 return ret;
1073 }
1074
GetMediaInfo(MediaInfo & mediaInfo)1075 Status FFmpegDemuxerPlugin::GetMediaInfo(MediaInfo& mediaInfo)
1076 {
1077 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetMediaInfo");
1078 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1079 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1080
1081 Status ret = GetSeiInfo();
1082 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "GetSeiInfo failed");
1083
1084 FFmpegFormatHelper::ParseMediaInfo(*formatContext_, mediaInfo.general);
1085 DemuxerLogCompressor::StringifyMeta(mediaInfo.general, -1); // source meta
1086 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1087 Meta meta;
1088 auto avStream = formatContext_->streams[trackIndex];
1089 if (avStream == nullptr) {
1090 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " info is nullptr", trackIndex);
1091 mediaInfo.tracks.push_back(meta);
1092 continue;
1093 }
1094 FFmpegFormatHelper::ParseTrackInfo(*avStream, meta, *formatContext_);
1095 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC) {
1096 if (streamParser_ != nullptr && streamParserInited_ && firstFrame_ != nullptr) {
1097 streamParser_->ConvertPacketToAnnexb(&(firstFrame_->data), firstFrame_->size, nullptr, 0, false);
1098 streamParser_->ParseAnnexbExtraData(firstFrame_->data, firstFrame_->size);
1099 // Parser only sends xps info when first call ConvertPacketToAnnexb
1100 // readSample will call ConvertPacketToAnnexb again, so rest here
1101 streamParser_->ResetXPSSendStatus();
1102 ParseHEVCMetadataInfo(*avStream, meta);
1103 } else {
1104 MEDIA_LOG_W("Parse hevc info failed");
1105 }
1106 }
1107 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC ||
1108 avStream->codecpar->codec_id == AV_CODEC_ID_H264 ||
1109 avStream->codecpar->codec_id == AV_CODEC_ID_VVC) {
1110 ConvertCsdToAnnexb(*avStream, meta);
1111 }
1112 mediaInfo.tracks.push_back(meta);
1113 DemuxerLogCompressor::StringifyMeta(meta, trackIndex);
1114 }
1115 return Status::OK;
1116 }
1117
GetUserMeta(std::shared_ptr<Meta> meta)1118 Status FFmpegDemuxerPlugin::GetUserMeta(std::shared_ptr<Meta> meta)
1119 {
1120 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetUserMeta");
1121 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1122 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1123 FALSE_RETURN_V_MSG_E(meta != nullptr, Status::ERROR_NULL_POINTER, "Meta is nullptr");
1124
1125 FFmpegFormatHelper::ParseUserMeta(*formatContext_, meta);
1126 return Status::OK;
1127 }
1128
ParseDrmInfo(const MetaDrmInfo * const metaDrmInfo,size_t drmInfoSize,std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1129 void FFmpegDemuxerPlugin::ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize,
1130 std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1131 {
1132 MEDIA_LOG_D("In");
1133 size_t infoCount = drmInfoSize / sizeof(MetaDrmInfo);
1134 for (size_t index = 0; index < infoCount; index++) {
1135 std::stringstream ssConverter;
1136 std::string uuid;
1137 for (uint32_t i = 0; i < metaDrmInfo[index].uuidLen; i++) {
1138 int32_t singleUuid = static_cast<int32_t>(metaDrmInfo[index].uuid[i]);
1139 ssConverter << std::hex << std::setfill('0') << std::setw(2) << singleUuid; // 2:w
1140 uuid = ssConverter.str();
1141 }
1142 drmInfo.insert({ uuid, std::vector<uint8_t>(metaDrmInfo[index].pssh,
1143 metaDrmInfo[index].pssh + metaDrmInfo[index].psshLen) });
1144 }
1145 }
1146
GetDrmInfo(std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1147 Status FFmpegDemuxerPlugin::GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1148 {
1149 MEDIA_LOG_D("In");
1150 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1151 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1152
1153 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1154 Meta meta;
1155 AVStream *avStream = formatContext_->streams[trackIndex];
1156 if (avStream == nullptr) {
1157 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " is nullptr", trackIndex);
1158 continue;
1159 }
1160 MEDIA_LOG_D("GetDrmInfo by stream side data");
1161 size_t drmInfoSize = 0;
1162 MetaDrmInfo *tmpDrmInfo = (MetaDrmInfo *)av_stream_get_side_data(avStream,
1163 AV_PKT_DATA_ENCRYPTION_INIT_INFO, &drmInfoSize);
1164 if (tmpDrmInfo != nullptr && drmInfoSize != 0) {
1165 ParseDrmInfo(tmpDrmInfo, drmInfoSize, drmInfo);
1166 }
1167 }
1168 return Status::OK;
1169 }
1170
ConvertCsdToAnnexb(const AVStream & avStream,Meta & format)1171 void FFmpegDemuxerPlugin::ConvertCsdToAnnexb(const AVStream& avStream, Meta &format)
1172 {
1173 uint8_t *extradata = avStream.codecpar->extradata;
1174 int32_t extradataSize = avStream.codecpar->extradata_size;
1175 if (HaveValidParser(avStream.codecpar->codec_id) && streamParser_ != nullptr && streamParserInited_) {
1176 streamParser_->ConvertPacketToAnnexb(&(extradata), extradataSize, nullptr, 0, true);
1177 } else if (avStream.codecpar->codec_id == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
1178 if (avbsfContext_->par_out->extradata != nullptr && avbsfContext_->par_out->extradata_size > 0) {
1179 extradata = avbsfContext_->par_out->extradata;
1180 extradataSize = avbsfContext_->par_out->extradata_size;
1181 }
1182 }
1183 if (extradata != nullptr && extradataSize > 0) {
1184 std::vector<uint8_t> extra(extradataSize);
1185 extra.assign(extradata, extradata + extradataSize);
1186 format.Set<Tag::MEDIA_CODEC_CONFIG>(extra);
1187 }
1188 }
1189
AddPacketToCacheQueue(AVPacket * pkt)1190 Status FFmpegDemuxerPlugin::AddPacketToCacheQueue(AVPacket *pkt)
1191 {
1192 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Pkt is nullptr");
1193 #ifdef BUILD_ENG_VERSION
1194 DumpParam dumpParam {DumpMode(DUMP_AVPACKET_OUTPUT & dumpMode_), pkt->data, pkt->stream_index, -1, pkt->size,
1195 avpacketIndex_++, pkt->pts, pkt->pos};
1196 Dump(dumpParam);
1197 #endif
1198 auto trackId = pkt->stream_index;
1199 Status ret = Status::OK;
1200 if (NeedCombineFrame(trackId) && !GetNextFrame(pkt->data, pkt->size) && cacheQueue_.HasCache(trackId)) {
1201 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
1202 if (cacheSamplePacket != nullptr) {
1203 cacheSamplePacket->pkts.push_back(pkt);
1204 }
1205 } else {
1206 std::shared_ptr<SamplePacket> cacheSamplePacket = std::make_shared<SamplePacket>();
1207 if (cacheSamplePacket != nullptr) {
1208 cacheSamplePacket->pkts.push_back(pkt);
1209 cacheSamplePacket->offset = 0;
1210 cacheQueue_.Push(static_cast<uint32_t>(trackId), cacheSamplePacket);
1211 ret = CheckCacheDataLimit(static_cast<uint32_t>(trackId));
1212 }
1213 }
1214 return ret;
1215 }
1216
GetVideoFirstKeyFrame(uint32_t trackIndex)1217 Status FFmpegDemuxerPlugin::GetVideoFirstKeyFrame(uint32_t trackIndex)
1218 {
1219 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1220 AVPacket *pkt = nullptr;
1221 Status ret = Status::OK;
1222 while (1) {
1223 if (pkt == nullptr) {
1224 pkt = av_packet_alloc();
1225 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1226 }
1227
1228 std::unique_lock<std::mutex> sLock(syncMutex_);
1229 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
1230 sLock.unlock();
1231 if (ffmpegRet < 0) {
1232 MEDIA_LOG_E("Call av_read_frame failed, ret:" PUBLIC_LOG_D32, ffmpegRet);
1233 av_packet_unref(pkt);
1234 break;
1235 }
1236 cacheQueue_.AddTrackQueue(pkt->stream_index);
1237 ret = AddPacketToCacheQueue(pkt);
1238 if (ret != Status::OK) {
1239 return ret;
1240 }
1241
1242 if (static_cast<uint32_t>(pkt->stream_index) == trackIndex) {
1243 firstFrame_ = av_packet_alloc();
1244 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1245 int avRet = av_new_packet(firstFrame_, pkt->size);
1246 FALSE_RETURN_V_MSG_E(avRet >= 0, Status::ERROR_INVALID_DATA, "Call av_new_packet failed");
1247 av_packet_copy_props(firstFrame_, pkt);
1248 memcpy_s(firstFrame_->data, pkt->size, pkt->data, pkt->size);
1249 break;
1250 }
1251 pkt = nullptr;
1252 }
1253 return ret;
1254 }
1255
ParseHEVCMetadataInfo(const AVStream & avStream,Meta & format)1256 void FFmpegDemuxerPlugin::ParseHEVCMetadataInfo(const AVStream& avStream, Meta& format)
1257 {
1258 HevcParseFormat parse;
1259 parse.isHdrVivid = streamParser_->IsHdrVivid();
1260 parse.colorRange = streamParser_->GetColorRange();
1261 parse.colorPrimaries = streamParser_->GetColorPrimaries();
1262 parse.colorTransfer = streamParser_->GetColorTransfer();
1263 parse.colorMatrixCoeff = streamParser_->GetColorMatrixCoeff();
1264 parse.profile = streamParser_->GetProfileIdc();
1265 parse.level = streamParser_->GetLevelIdc();
1266 parse.chromaLocation = streamParser_->GetChromaLocation();
1267 parse.picWidInLumaSamples = streamParser_->GetPicWidInLumaSamples();
1268 parse.picHetInLumaSamples = streamParser_->GetPicHetInLumaSamples();
1269
1270 FFmpegFormatHelper::ParseHevcInfo(*formatContext_, parse, format);
1271 }
1272
TrackIsSelected(const uint32_t trackId)1273 bool FFmpegDemuxerPlugin::TrackIsSelected(const uint32_t trackId)
1274 {
1275 return std::any_of(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1276 [trackId](uint32_t id) { return id == trackId; });
1277 }
1278
SelectTrack(uint32_t trackId)1279 Status FFmpegDemuxerPlugin::SelectTrack(uint32_t trackId)
1280 {
1281 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1282 MEDIA_LOG_I("Select " PUBLIC_LOG_D32, trackId);
1283 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1284 if (trackId >= static_cast<uint32_t>(formatContext_.get()->nb_streams)) {
1285 MEDIA_LOG_E("Track is invalid, just have " PUBLIC_LOG_D32 " tracks in file", formatContext_.get()->nb_streams);
1286 return Status::ERROR_INVALID_PARAMETER;
1287 }
1288
1289 AVStream* avStream = formatContext_->streams[trackId];
1290 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1291 if (!IsSupportedTrack(*avStream)) {
1292 MEDIA_LOG_E("Track type is unsupport");
1293 return Status::ERROR_INVALID_PARAMETER;
1294 }
1295
1296 if (!TrackIsSelected(trackId)) {
1297 selectedTrackIds_.push_back(trackId);
1298 trackMtx_[trackId] = std::make_shared<std::mutex>();
1299 trackDfxInfoMap_[trackId] = {0, -1, -1};
1300 return cacheQueue_.AddTrackQueue(trackId);
1301 } else {
1302 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " has been selected", trackId);
1303 }
1304 return Status::OK;
1305 }
1306
UnselectTrack(uint32_t trackId)1307 Status FFmpegDemuxerPlugin::UnselectTrack(uint32_t trackId)
1308 {
1309 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1310 MEDIA_LOG_I("Unselect " PUBLIC_LOG_D32, trackId);
1311 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1312 auto index = std::find_if(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1313 [trackId](uint32_t selectedId) {return trackId == selectedId; });
1314 if (TrackIsSelected(trackId)) {
1315 selectedTrackIds_.erase(index);
1316 trackMtx_.erase(trackId);
1317 trackDfxInfoMap_.erase(trackId);
1318 return cacheQueue_.RemoveTrackQueue(trackId);
1319 } else {
1320 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " is not in selected list", trackId);
1321 }
1322 return Status::OK;
1323 }
1324
SeekTo(int32_t trackId,int64_t seekTime,SeekMode mode,int64_t & realSeekTime)1325 Status FFmpegDemuxerPlugin::SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime)
1326 {
1327 (void) trackId;
1328 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1329 MediaAVCodec::AVCodecTrace trace("SeekTo");
1330 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1331 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1332
1333 FALSE_RETURN_V_MSG_E(seekTime >= 0, Status::ERROR_INVALID_PARAMETER,
1334 "Seek time " PUBLIC_LOG_D64 " is not unsupported", seekTime);
1335 FALSE_RETURN_V_MSG_E(g_seekModeToFFmpegSeekFlags.count(mode) != 0, Status::ERROR_INVALID_PARAMETER,
1336 "Seek mode " PUBLIC_LOG_D32 " is not unsupported", static_cast<uint32_t>(mode));
1337
1338 int trackIndex = static_cast<int>(selectedTrackIds_[0]);
1339 for (size_t i = 1; i < selectedTrackIds_.size(); i++) {
1340 int index = static_cast<int>(selectedTrackIds_[i]);
1341 if (formatContext_->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
1342 trackIndex = index;
1343 break;
1344 }
1345 }
1346 MEDIA_LOG_D("Seek based on track " PUBLIC_LOG_D32, trackIndex);
1347 auto avStream = formatContext_->streams[trackIndex];
1348 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1349 int64_t ffTime = ConvertTimeToFFmpeg(seekTime * 1000 * 1000, avStream->time_base);
1350 if (!CheckStartTime(formatContext_.get(), avStream, ffTime, seekTime)) {
1351 MEDIA_LOG_E("Get start time from track " PUBLIC_LOG_D32 " failed", trackIndex);
1352 return Status::ERROR_INVALID_OPERATION;
1353 }
1354 realSeekTime = ConvertTimeFromFFmpeg(ffTime, avStream->time_base);
1355 int flag = ConvertFlagsToFFmpeg(avStream, ffTime, mode, seekTime);
1356 MEDIA_LOG_I("Time [" PUBLIC_LOG_U64 "/" PUBLIC_LOG_U64 "/" PUBLIC_LOG_D64 "] flag ["
1357 PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
1358 seekTime, ffTime, realSeekTime, static_cast<int32_t>(mode), flag);
1359 auto ret = av_seek_frame(formatContext_.get(), trackIndex, ffTime, flag);
1360 if (formatContext_->pb->error) {
1361 formatContext_->pb->error = 0;
1362 }
1363 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
1364 "Call av_seek_frame failed, err: " PUBLIC_LOG_S, AVStrError(ret).c_str());
1365 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1366 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1367 cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1368 }
1369 return Status::OK;
1370 }
1371
Flush()1372 Status FFmpegDemuxerPlugin::Flush()
1373 {
1374 Status ret = Status::OK;
1375 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1376 MEDIA_LOG_I("In");
1377 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1378 ret = cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1379 ret = cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1380 }
1381 if (formatContext_) {
1382 avio_flush(formatContext_.get()->pb);
1383 avformat_flush(formatContext_.get());
1384 }
1385 return ret;
1386 }
1387
ResetEosStatus()1388 void FFmpegDemuxerPlugin::ResetEosStatus()
1389 {
1390 MEDIA_LOG_I("In");
1391 formatContext_->pb->eof_reached = 0;
1392 formatContext_->pb->error = 0;
1393 }
1394
ReadSample(uint32_t trackId,std::shared_ptr<AVBuffer> sample)1395 Status FFmpegDemuxerPlugin::ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample)
1396 {
1397 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1398 MediaAVCodec::AVCodecTrace trace("ReadSample");
1399 MEDIA_LOG_D("In");
1400 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1401 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1402 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_INVALID_PARAMETER, "Track has not been selected");
1403 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_!=nullptr, Status::ERROR_INVALID_PARAMETER,
1404 "AVBuffer or memory is nullptr");
1405 Status ret;
1406 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1407 ret = ReadPacketToCacheQueue(trackId);
1408 }
1409 while (!cacheQueue_.HasCache(trackId)) {
1410 ret = ReadPacketToCacheQueue(trackId);
1411 if (ret == Status::END_OF_STREAM) {
1412 MEDIA_LOG_D("Read to end");
1413 }
1414 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1415 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1416 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1417 }
1418 std::lock_guard<std::mutex> lockTrack(*trackMtx_[trackId].get());
1419 auto samplePacket = cacheQueue_.Front(trackId);
1420 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_NULL_POINTER, "Cache packet is nullptr");
1421 if (samplePacket->isEOS) {
1422 ret = SetEosSample(sample);
1423 if (ret == Status::OK) {
1424 MEDIA_LOG_I("Track:" PUBLIC_LOG_D32 " eos [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
1425 trackId, trackDfxInfoMap_[trackId].lastPts,
1426 trackDfxInfoMap_[trackId].lastDurantion, trackDfxInfoMap_[trackId].lastPos);
1427 cacheQueue_.Pop(trackId);
1428 }
1429 return ret;
1430 }
1431 ret = ConvertAVPacketToSample(sample, samplePacket);
1432 if (ret == Status::ERROR_NOT_ENOUGH_DATA) {
1433 return Status::OK;
1434 } else if (ret == Status::OK) {
1435 MEDIA_LOG_D("All partial sample has been copied");
1436 cacheQueue_.Pop(trackId);
1437 }
1438 return ret;
1439 }
1440
GetNextSampleSize(uint32_t trackId,int32_t & size)1441 Status FFmpegDemuxerPlugin::GetNextSampleSize(uint32_t trackId, int32_t& size)
1442 {
1443 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1444 MediaAVCodec::AVCodecTrace trace("GetNextSampleSize");
1445 MEDIA_LOG_D("In, track " PUBLIC_LOG_D32, trackId);
1446 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1447 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_UNKNOWN, "Track has not been selected");
1448
1449 Status ret;
1450 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1451 ret = ReadPacketToCacheQueue(trackId);
1452 }
1453 while (!cacheQueue_.HasCache(trackId)) {
1454 ret = ReadPacketToCacheQueue(trackId);
1455 if (ret == Status::END_OF_STREAM) {
1456 MEDIA_LOG_D("Read to end");
1457 }
1458 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1459 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1460 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1461 }
1462 std::shared_ptr<SamplePacket> samplePacket = cacheQueue_.Front(trackId);
1463 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_UNKNOWN, "Cache sample is nullptr");
1464 if (samplePacket->isEOS) {
1465 MEDIA_LOG_I("Track " PUBLIC_LOG_D32 " eos", trackId);
1466 return Status::END_OF_STREAM;
1467 }
1468 FALSE_RETURN_V_MSG_E(samplePacket->pkts.size() > 0, Status::ERROR_UNKNOWN, "Cache sample is empty");
1469 int totalSize = 0;
1470 for (auto pkt : samplePacket->pkts) {
1471 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_UNKNOWN, "Packet in sample is nullptr");
1472 totalSize += pkt->size;
1473 }
1474 size = totalSize;
1475 return Status::OK;
1476 }
1477
InitPTSandIndexConvert()1478 void FFmpegDemuxerPlugin::InitPTSandIndexConvert()
1479 {
1480 indexToRelativePTSFrameCount_ = 0; // init IndexToRelativePTSFrameCount_
1481 relativePTSToIndexPosition_ = 0; // init RelativePTSToIndexPosition_
1482 indexToRelativePTSMaxHeap_ = std::priority_queue<int64_t>(); // init IndexToRelativePTSMaxHeap_
1483 relativePTSToIndexPTSMin_ = INT64_MAX;
1484 relativePTSToIndexPTSMax_ = INT64_MIN;
1485 relativePTSToIndexRightDiff_ = INT64_MAX;
1486 relativePTSToIndexLeftDiff_ = INT64_MAX;
1487 relativePTSToIndexTempDiff_ = INT64_MAX;
1488 }
1489
GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,const uint64_t relativePresentationTimeUs,uint32_t & index)1490 Status FFmpegDemuxerPlugin::GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
1491 const uint64_t relativePresentationTimeUs, uint32_t &index)
1492 {
1493 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1494
1495 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1496
1497 InitPTSandIndexConvert();
1498
1499 auto avStream = formatContext_->streams[trackIndex];
1500 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1501
1502 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1503 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1504
1505 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1506 static_cast<int64_t>(relativePresentationTimeUs), index);
1507 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1508
1509 int64_t absolutePTS = static_cast<int64_t>(relativePresentationTimeUs) + absolutePTSIndexZero_;
1510
1511 ret = GetPresentationTimeUsFromFfmpegMOV(RELATIVEPTS_TO_INDEX, trackIndex,
1512 absolutePTS, index);
1513 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1514
1515 if (absolutePTS < relativePTSToIndexPTSMin_ || absolutePTS > relativePTSToIndexPTSMax_) {
1516 MEDIA_LOG_E("Pts is out of range");
1517 return Status::ERROR_INVALID_DATA;
1518 }
1519
1520 if (relativePTSToIndexLeftDiff_ == 0 || relativePTSToIndexRightDiff_ == 0) {
1521 index = relativePTSToIndexPosition_;
1522 } else {
1523 index = relativePTSToIndexLeftDiff_ < relativePTSToIndexRightDiff_ ?
1524 relativePTSToIndexPosition_ - 1 : relativePTSToIndexPosition_;
1525 }
1526 return Status::OK;
1527 }
1528
GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,const uint32_t index,uint64_t & relativePresentationTimeUs)1529 Status FFmpegDemuxerPlugin::GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
1530 const uint32_t index, uint64_t &relativePresentationTimeUs)
1531 {
1532 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1533
1534 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1535
1536 InitPTSandIndexConvert();
1537
1538 auto avStream = formatContext_->streams[trackIndex];
1539 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1540
1541 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1542 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1543
1544 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1545 static_cast<int64_t>(relativePresentationTimeUs), index);
1546 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1547
1548 GetPresentationTimeUsFromFfmpegMOV(INDEX_TO_RELATIVEPTS, trackIndex,
1549 static_cast<int64_t>(relativePresentationTimeUs), index);
1550 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1551
1552 if (index + 1 > indexToRelativePTSFrameCount_) {
1553 MEDIA_LOG_E("Index is out of range");
1554 return Status::ERROR_INVALID_DATA;
1555 }
1556
1557 int64_t relativepts = indexToRelativePTSMaxHeap_.top() - absolutePTSIndexZero_;
1558 FALSE_RETURN_V_MSG_E(relativepts >= 0, Status::ERROR_INVALID_DATA, "Existence of calculation results less than 0");
1559 relativePresentationTimeUs = static_cast<uint64_t>(relativepts);
1560
1561 return Status::OK;
1562 }
1563
PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1564 Status FFmpegDemuxerPlugin::PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,
1565 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1566 {
1567 uint32_t sttsIndex = 0;
1568 uint32_t cttsIndex = 0;
1569 int64_t pts = 0; // init pts
1570 int64_t dts = 0; // init dts
1571
1572 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1573 int32_t cttsCurNum = 0;
1574
1575 cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1576 while (sttsIndex < avStream->stts_count && cttsIndex < avStream->ctts_count &&
1577 cttsCurNum >= 0 && sttsCurNum >= 0) {
1578 if (cttsCurNum == 0) {
1579 cttsIndex++;
1580 cttsCurNum = cttsIndex < avStream->ctts_count ?
1581 static_cast<int32_t>(avStream->ctts_data[cttsIndex].count) : 0;
1582 }
1583 cttsCurNum--;
1584 pts = (dts + static_cast<int64_t>(avStream->ctts_data[cttsIndex].duration)) *
1585 1000 * 1000 / static_cast<int64_t>(avStream->time_scale); // 1000 is used for converting pts to us
1586 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1587 sttsCurNum--;
1588 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1589 if (sttsCurNum == 0) {
1590 sttsIndex++;
1591 sttsCurNum = sttsIndex < avStream->stts_count ?
1592 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1593 }
1594 }
1595 return Status::OK;
1596 }
1597
PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1598 Status FFmpegDemuxerPlugin::PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,
1599 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1600 {
1601 uint32_t sttsIndex = 0;
1602 int64_t pts = 0; // init pts
1603 int64_t dts = 0; // init dts
1604
1605 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1606
1607 while (sttsIndex < avStream->stts_count && sttsCurNum >= 0) {
1608 pts = dts * 1000 * 1000 / static_cast<int64_t>(avStream->time_scale); // 1000 is for converting pts to us
1609 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1610 sttsCurNum--;
1611 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1612 if (sttsCurNum == 0) {
1613 sttsIndex++;
1614 sttsCurNum = sttsIndex < avStream->stts_count ?
1615 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1616 }
1617 }
1618 return Status::OK;
1619 }
1620
GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,uint32_t trackIndex,int64_t absolutePTS,uint32_t index)1621 Status FFmpegDemuxerPlugin::GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,
1622 uint32_t trackIndex, int64_t absolutePTS, uint32_t index)
1623 {
1624 auto avStream = formatContext_->streams[trackIndex];
1625 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1626 FALSE_RETURN_V_MSG_E(avStream->stts_data != nullptr && avStream->stts_count != 0,
1627 Status::ERROR_NULL_POINTER, "AVStream->stts_data is empty");
1628 FALSE_RETURN_V_MSG_E(avStream->time_scale != 0, Status::ERROR_INVALID_DATA, "AVStream->time_scale is zero");
1629 if (mode == GET_ALL_FRAME_PTS) {
1630 ptsListOrg_.clear();
1631 }
1632 return avStream->ctts_data == nullptr ?
1633 PTSAndIndexConvertOnlySttsProcess(mode, avStream, absolutePTS, index) :
1634 PTSAndIndexConvertSttsAndCttsProcess(mode, avStream, absolutePTS, index);
1635 }
1636
PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,int64_t pts,int64_t absolutePTS,uint32_t index)1637 void FFmpegDemuxerPlugin::PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,
1638 int64_t pts, int64_t absolutePTS, uint32_t index)
1639 {
1640 switch (mode) {
1641 case GET_FIRST_PTS:
1642 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1643 break;
1644 case INDEX_TO_RELATIVEPTS:
1645 IndexToRelativePTSProcess(pts, index);
1646 break;
1647 case RELATIVEPTS_TO_INDEX:
1648 RelativePTSToIndexProcess(pts, absolutePTS);
1649 break;
1650 case GET_ALL_FRAME_PTS:
1651 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1652 ptsListOrg_.emplace_back(pts);
1653 break;
1654 default:
1655 MEDIA_LOG_E("Wrong mode");
1656 break;
1657 }
1658 }
1659
IndexToRelativePTSProcess(int64_t pts,uint32_t index)1660 void FFmpegDemuxerPlugin::IndexToRelativePTSProcess(int64_t pts, uint32_t index)
1661 {
1662 if (indexToRelativePTSMaxHeap_.size() < index + 1) {
1663 indexToRelativePTSMaxHeap_.push(pts);
1664 } else {
1665 if (pts < indexToRelativePTSMaxHeap_.top()) {
1666 indexToRelativePTSMaxHeap_.pop();
1667 indexToRelativePTSMaxHeap_.push(pts);
1668 }
1669 }
1670 indexToRelativePTSFrameCount_++;
1671 }
1672
RelativePTSToIndexProcess(int64_t pts,int64_t absolutePTS)1673 void FFmpegDemuxerPlugin::RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS)
1674 {
1675 if (relativePTSToIndexPTSMin_ > pts) {
1676 relativePTSToIndexPTSMin_ = pts;
1677 }
1678 if (relativePTSToIndexPTSMax_ < pts) {
1679 relativePTSToIndexPTSMax_ = pts;
1680 }
1681 relativePTSToIndexTempDiff_ = abs(pts - absolutePTS);
1682 if (pts < absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexLeftDiff_) {
1683 relativePTSToIndexLeftDiff_ = relativePTSToIndexTempDiff_;
1684 }
1685 if (pts >= absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexRightDiff_) {
1686 relativePTSToIndexRightDiff_ = relativePTSToIndexTempDiff_;
1687 }
1688 if (pts < absolutePTS) {
1689 relativePTSToIndexPosition_++;
1690 }
1691 }
1692
CheckCacheDataLimit(uint32_t trackId)1693 Status FFmpegDemuxerPlugin::CheckCacheDataLimit(uint32_t trackId)
1694 {
1695 if (!outOfLimit_) {
1696 auto cacheDataSize = cacheQueue_.GetCacheDataSize(trackId);
1697 if (cacheDataSize > cachelimitSize_) {
1698 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " cache out of limit: " PUBLIC_LOG_U32 "/" PUBLIC_LOG_U32 ", by user "
1699 PUBLIC_LOG_D32, trackId, cacheDataSize, cachelimitSize_, static_cast<int32_t>(setLimitByUser));
1700 outOfLimit_ = true;
1701 }
1702 }
1703 return Status::OK;
1704 }
1705
SetCacheLimit(uint32_t limitSize)1706 void FFmpegDemuxerPlugin::SetCacheLimit(uint32_t limitSize)
1707 {
1708 setLimitByUser = true;
1709 cachelimitSize_ = limitSize;
1710 }
1711
1712 namespace { // plugin set
1713
IsStartWithID3(const uint8_t * buf,const char * tagName)1714 int IsStartWithID3(const uint8_t *buf, const char *tagName)
1715 {
1716 return buf[POS_0] == tagName[POS_0] &&
1717 buf[POS_1] == tagName[POS_1] &&
1718 buf[POS_2] == tagName[POS_2] &&
1719 buf[POS_3] != POS_FF &&
1720 buf[POS_4] != POS_FF &&
1721 (buf[POS_6] & TAG_MASK) == 0 &&
1722 (buf[POS_7] & TAG_MASK) == 0 &&
1723 (buf[POS_8] & TAG_MASK) == 0 &&
1724 (buf[POS_9] & TAG_MASK) == 0;
1725 }
1726
GetID3TagLen(const uint8_t * buf)1727 int GetID3TagLen(const uint8_t *buf)
1728 {
1729 int32_t len = ((buf[POS_6] & LEN_MASK) << POS_21) + ((buf[POS_7] & LEN_MASK) << POS_14) +
1730 ((buf[POS_8] & LEN_MASK) << POS_7) + (buf[POS_9] & LEN_MASK) +
1731 static_cast<int32_t>(ID3V2_HEADER_SIZE);
1732 if (buf[POS_5] & TAG_VERSION_MASK) {
1733 len += static_cast<int32_t>(ID3V2_HEADER_SIZE);
1734 }
1735 return len;
1736 }
1737
GetConfidence(std::shared_ptr<AVInputFormat> plugin,const std::string & pluginName,std::shared_ptr<DataSource> dataSource,size_t & getData)1738 int32_t GetConfidence(std::shared_ptr<AVInputFormat> plugin, const std::string& pluginName,
1739 std::shared_ptr<DataSource> dataSource, size_t &getData)
1740 {
1741 size_t bufferSize = DEFAULT_SNIFF_SIZE;
1742 uint64_t fileSize = 0;
1743 Status getFileSize = dataSource->GetSize(fileSize);
1744 if (getFileSize == Status::OK) {
1745 bufferSize = (bufferSize < fileSize) ? bufferSize : fileSize;
1746 }
1747 std::vector<uint8_t> buff(bufferSize + AVPROBE_PADDING_SIZE); // fix ffmpeg probe crash, refer to tools/probetest.c
1748 auto bufferInfo = std::make_shared<Buffer>();
1749 auto bufData = bufferInfo->WrapMemory(buff.data(), bufferSize, bufferSize);
1750 FALSE_RETURN_V_MSG_E(bufferInfo->GetMemory() != nullptr, -1,
1751 "Alloc buffer failed for " PUBLIC_LOG_S, pluginName.c_str());
1752 Status ret = Status::OK;
1753 {
1754 std::string traceName = "Sniff_" + pluginName + "_Readat";
1755 MediaAVCodec::AVCodecTrace trace(traceName.c_str());
1756 ret = dataSource->ReadAt(0, bufferInfo, bufferSize);
1757 }
1758 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1759 getData = bufferInfo->GetMemory()->GetSize();
1760 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1761 if (getFileSize == Status::OK && getData > ID3V2_HEADER_SIZE && IsStartWithID3(buff.data(), "ID3")) {
1762 int32_t id3Len = GetID3TagLen(buff.data());
1763 // id3 tag length is out of file, or file just contains id3 tag, no valid data.
1764 FALSE_RETURN_V_MSG_E(id3Len >= 0 && static_cast<uint64_t>(id3Len) < fileSize, -1,
1765 "File data error for " PUBLIC_LOG_S, pluginName.c_str());
1766 if (id3Len > 0) {
1767 uint64_t remainSize = fileSize - static_cast<uint64_t>(id3Len);
1768 bufferSize = (bufferSize < remainSize) ? bufferSize : remainSize;
1769 int resetRet = memset_s(buff.data(), bufferSize, 0, bufferSize);
1770 FALSE_RETURN_V_MSG_E(resetRet == EOK, -1, "Reset buff failed for " PUBLIC_LOG_S, pluginName.c_str());
1771 ret = dataSource->ReadAt(id3Len, bufferInfo, bufferSize);
1772 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1773 getData = bufferInfo->GetMemory()->GetSize();
1774 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1775 }
1776 }
1777 AVProbeData probeData{"", buff.data(), static_cast<int32_t>(getData), ""};
1778 return plugin->read_probe(&probeData);
1779 }
1780
Sniff(const std::string & pluginName,std::shared_ptr<DataSource> dataSource)1781 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource)
1782 {
1783 FALSE_RETURN_V_MSG_E(!pluginName.empty(), 0, "Plugin name is empty");
1784 FALSE_RETURN_V_MSG_E(dataSource != nullptr, 0, "DataSource is nullptr");
1785 std::shared_ptr<AVInputFormat> plugin;
1786 {
1787 std::lock_guard<std::mutex> lock(g_mtx);
1788 plugin = g_pluginInputFormat[pluginName];
1789 }
1790 FALSE_RETURN_V_MSG_E((plugin != nullptr && plugin->read_probe), 0,
1791 "Get plugin for " PUBLIC_LOG_S " failed", pluginName.c_str());
1792 size_t getData = 0;
1793 int confidence = GetConfidence(plugin, pluginName, dataSource, getData);
1794 if (confidence < 0) {
1795 return 0;
1796 }
1797 if (StartWith(plugin->name, "mp3") && confidence > 0 && confidence <= MP3_PROBE_SCORE_LIMIT) {
1798 MEDIA_LOG_W("Score " PUBLIC_LOG_D32 " is too low", confidence);
1799 confidence = 0;
1800 }
1801 if (getData < DEFAULT_SNIFF_SIZE || confidence > 0) {
1802 MEDIA_LOG_I("Sniff:" PUBLIC_LOG_S "[" PUBLIC_LOG_ZU "/" PUBLIC_LOG_D32 "]", plugin->name, getData, confidence);
1803 }
1804 return confidence;
1805 }
1806
ReplaceDelimiter(const std::string & delmiters,char newDelimiter,std::string & str)1807 void ReplaceDelimiter(const std::string& delmiters, char newDelimiter, std::string& str)
1808 {
1809 MEDIA_LOG_D("Reset from [" PUBLIC_LOG_S "]", str.c_str());
1810 for (auto it = str.begin(); it != str.end(); ++it) {
1811 if (delmiters.find(newDelimiter) != std::string::npos) {
1812 *it = newDelimiter;
1813 }
1814 }
1815 MEDIA_LOG_D("Reset to [" PUBLIC_LOG_S "]", str.c_str());
1816 };
1817
RegisterPlugins(const std::shared_ptr<Register> & reg)1818 Status RegisterPlugins(const std::shared_ptr<Register>& reg)
1819 {
1820 MEDIA_LOG_I("In");
1821 FALSE_RETURN_V_MSG_E(reg != nullptr, Status::ERROR_INVALID_PARAMETER, "Register is nullptr");
1822 std::lock_guard<std::mutex> lock(g_mtx);
1823 const AVInputFormat* plugin = nullptr;
1824 void* i = nullptr;
1825 while ((plugin = av_demuxer_iterate(&i))) {
1826 if (plugin == nullptr) {
1827 continue;
1828 }
1829 MEDIA_LOG_D("Check ffmpeg demuxer " PUBLIC_LOG_S "[" PUBLIC_LOG_S "]", plugin->name, plugin->long_name);
1830 if (plugin->long_name != nullptr &&
1831 !strncmp(plugin->long_name, "pcm ", STR_MAX_LEN)) {
1832 continue;
1833 }
1834 if (!IsInputFormatSupported(plugin->name)) {
1835 continue;
1836 }
1837
1838 std::string pluginName = "avdemux_" + std::string(plugin->name);
1839 ReplaceDelimiter(".,|-<> ", '_', pluginName);
1840
1841 DemuxerPluginDef regInfo;
1842 regInfo.name = pluginName;
1843 regInfo.description = "ffmpeg demuxer plugin";
1844 regInfo.rank = RANK_MAX;
1845 regInfo.AddExtensions(SplitString(plugin->extensions, ','));
1846 g_pluginInputFormat[pluginName] =
1847 std::shared_ptr<AVInputFormat>(const_cast<AVInputFormat*>(plugin), [](void*) {});
1848 auto func = [](const std::string& name) -> std::shared_ptr<DemuxerPlugin> {
1849 return std::make_shared<FFmpegDemuxerPlugin>(name);
1850 };
1851 regInfo.SetCreator(func);
1852 regInfo.SetSniffer(Sniff);
1853 auto ret = reg->AddPlugin(regInfo);
1854 if (ret != Status::OK) {
1855 MEDIA_LOG_E("Add plugin failed, err=" PUBLIC_LOG_D32, static_cast<int>(ret));
1856 } else {
1857 MEDIA_LOG_D("Add plugin " PUBLIC_LOG_S, pluginName.c_str());
1858 }
1859 }
1860 FALSE_RETURN_V_MSG_E(!g_pluginInputFormat.empty(), Status::ERROR_UNKNOWN, "Can not load any ffmpeg demuxer");
1861 return Status::OK;
1862 }
1863 } // namespace
__anond76ed70e0a02null1864 PLUGIN_DEFINITION(FFmpegDemuxer, LicenseType::LGPL, RegisterPlugins, [] {});
1865 } // namespace Ffmpeg
1866 } // namespace Plugins
1867 } // namespace Media
1868 } // namespace OHOS
1869