1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #define MEDIA_PLUGIN
17 #define HST_LOG_TAG "FfmpegDemuxerPlugin"
18 #include <unistd.h>
19 #include <algorithm>
20 #include <malloc.h>
21 #include <string>
22 #include <sstream>
23 #include <iomanip>
24 #include <map>
25 #include <fstream>
26 #include <chrono>
27 #include <limits>
28 #include "avcodec_trace.h"
29 #include "securec.h"
30 #include "ffmpeg_format_helper.h"
31 #include "ffmpeg_utils.h"
32 #include "buffer/avbuffer.h"
33 #include "plugin/plugin_buffer.h"
34 #include "plugin/plugin_definition.h"
35 #include "common/log.h"
36 #include "meta/video_types.h"
37 #include "demuxer_log_compressor.h"
38 #include "ffmpeg_demuxer_plugin.h"
39 #include "meta/format.h"
40 #include "syspara/parameters.h"
41
42 namespace {
43 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_DEMUXER, "FfmpegDemuxerPlugin" };
44 }
45
46 namespace OHOS {
47 namespace Media {
48 namespace Plugins {
49 namespace Ffmpeg {
50 const uint32_t DEFAULT_READ_SIZE = 4096;
51 const uint32_t DEFAULT_SNIFF_SIZE = 4096 * 4;
52 const int32_t MP3_PROBE_SCORE_LIMIT = 5;
53 const uint32_t STR_MAX_LEN = 4;
54 const uint32_t RANK_MAX = 100;
55 const uint32_t NAL_START_CODE_SIZE = 4;
56 const uint32_t INIT_DOWNLOADS_DATA_SIZE_THRESHOLD = 2 * 1024 * 1024;
57 const int64_t LIVE_FLV_PROBE_SIZE = 100 * 1024 * 2;
58 const uint32_t DEFAULT_CACHE_LIMIT = 50 * 1024 * 1024; // 50M
59 const int64_t INIT_TIME_THRESHOLD = 1000;
60 const uint32_t ID3V2_HEADER_SIZE = 10;
61 const int32_t MS_TO_NS = 1000 * 1000;
62 const uint32_t REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT = 200000;
63
64 // id3v2 tag position
65 const int32_t POS_0 = 0;
66 const int32_t POS_1 = 1;
67 const int32_t POS_2 = 2;
68 const int32_t POS_3 = 3;
69 const int32_t POS_4 = 4;
70 const int32_t POS_5 = 5;
71 const int32_t POS_6 = 6;
72 const int32_t POS_7 = 7;
73 const int32_t POS_8 = 8;
74 const int32_t POS_9 = 9;
75 const int32_t POS_14 = 14;
76 const int32_t POS_21 = 21;
77 const int32_t POS_FF = 0xff;
78 const int32_t LEN_MASK = 0x7f;
79 const int32_t TAG_MASK = 0x80;
80 const int32_t TAG_VERSION_MASK = 0x10;
81
82 namespace {
83 std::map<std::string, std::shared_ptr<AVInputFormat>> g_pluginInputFormat;
84 std::mutex g_mtx;
85
86 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource);
87
88 Status RegisterPlugins(const std::shared_ptr<Register>& reg);
89
90 void ReplaceDelimiter(const std::string &delmiters, char newDelimiter, std::string &str);
91
92 static const std::map<SeekMode, int32_t> g_seekModeToFFmpegSeekFlags = {
93 { SeekMode::SEEK_PREVIOUS_SYNC, AVSEEK_FLAG_BACKWARD },
94 { SeekMode::SEEK_NEXT_SYNC, AVSEEK_FLAG_FRAME },
95 { SeekMode::SEEK_CLOSEST_SYNC, AVSEEK_FLAG_FRAME | AVSEEK_FLAG_BACKWARD }
96 };
97
98 static const std::map<AVCodecID, std::string> g_bitstreamFilterMap = {
99 { AV_CODEC_ID_H264, "h264_mp4toannexb" },
100 };
101
102 static const std::map<AVCodecID, StreamType> g_streamParserMap = {
103 { AV_CODEC_ID_HEVC, StreamType::HEVC },
104 { AV_CODEC_ID_VVC, StreamType::VVC },
105 };
106
107 static const std::vector<AVMediaType> g_streamMediaTypeVec = {
108 AVMEDIA_TYPE_AUDIO,
109 AVMEDIA_TYPE_VIDEO,
110 AVMEDIA_TYPE_SUBTITLE,
111 AVMEDIA_TYPE_TIMEDMETA
112 };
113
114 static std::vector<AVCodecID> g_imageCodecID = {
115 AV_CODEC_ID_MJPEG,
116 AV_CODEC_ID_PNG,
117 AV_CODEC_ID_PAM,
118 AV_CODEC_ID_BMP,
119 AV_CODEC_ID_JPEG2000,
120 AV_CODEC_ID_TARGA,
121 AV_CODEC_ID_TIFF,
122 AV_CODEC_ID_GIF,
123 AV_CODEC_ID_PCX,
124 AV_CODEC_ID_XWD,
125 AV_CODEC_ID_XBM,
126 AV_CODEC_ID_WEBP,
127 AV_CODEC_ID_APNG,
128 AV_CODEC_ID_XPM,
129 AV_CODEC_ID_SVG,
130 };
131
HaveValidParser(const AVCodecID codecId)132 bool HaveValidParser(const AVCodecID codecId)
133 {
134 return g_streamParserMap.count(codecId) != 0;
135 }
136
GetFileDuration(const AVFormatContext & avFormatContext)137 int64_t GetFileDuration(const AVFormatContext& avFormatContext)
138 {
139 int64_t duration = 0;
140 const AVDictionaryEntry *metaDuration = av_dict_get(avFormatContext.metadata, "DURATION", NULL, 0);
141 int64_t us;
142 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
143 if (us > duration) {
144 MEDIA_LOG_D("Get duration from file");
145 duration = us;
146 }
147 }
148
149 if (duration <= 0) {
150 for (uint32_t i = 0; i < avFormatContext.nb_streams; ++i) {
151 auto streamDuration = (ConvertTimeFromFFmpeg(avFormatContext.streams[i]->duration,
152 avFormatContext.streams[i]->time_base)) / 1000; // us
153 if (streamDuration > duration) {
154 MEDIA_LOG_D("Get duration from stream " PUBLIC_LOG_U32, i);
155 duration = streamDuration;
156 }
157 }
158 }
159 return duration;
160 }
161
GetStreamDuration(const AVStream & avStream)162 int64_t GetStreamDuration(const AVStream& avStream)
163 {
164 int64_t duration = 0;
165 const AVDictionaryEntry *metaDuration = av_dict_get(avStream.metadata, "DURATION", NULL, 0);
166 int64_t us;
167 if (metaDuration != nullptr && (av_parse_time(&us, metaDuration->value, 1) == 0)) {
168 if (us > duration) {
169 MEDIA_LOG_D("Get duration from stream");
170 duration = us;
171 }
172 }
173 return duration;
174 }
175
CheckStartTime(const AVFormatContext * formatContext,const AVStream * stream,int64_t & timeStamp,int64_t seekTime)176 bool CheckStartTime(const AVFormatContext *formatContext, const AVStream *stream, int64_t &timeStamp, int64_t seekTime)
177 {
178 int64_t startTime = 0;
179 int64_t num = 1000; // ms convert us
180 FALSE_RETURN_V_MSG_E(stream != nullptr, false, "String is nullptr");
181 if (stream->start_time != AV_NOPTS_VALUE) {
182 startTime = stream->start_time;
183 if (timeStamp > 0 && startTime > INT64_MAX - timeStamp) {
184 MEDIA_LOG_E("Seek value overflow with start time: " PUBLIC_LOG_D64 " timeStamp: " PUBLIC_LOG_D64,
185 startTime, timeStamp);
186 return false;
187 }
188 }
189 MEDIA_LOG_D("StartTime: " PUBLIC_LOG_D64, startTime);
190 int64_t fileDuration = formatContext->duration;
191 int64_t streamDuration = stream->duration;
192 if (fileDuration == AV_NOPTS_VALUE || fileDuration <= 0) {
193 fileDuration = GetFileDuration(*formatContext);
194 }
195 if (streamDuration == AV_NOPTS_VALUE || streamDuration <= 0) {
196 streamDuration = GetStreamDuration(*stream);
197 }
198 MEDIA_LOG_D("File duration=" PUBLIC_LOG_D64 ", stream duration=" PUBLIC_LOG_D64, fileDuration, streamDuration);
199 // when timestemp out of file duration, return error
200 if (fileDuration > 0 && seekTime * num > fileDuration) { // fileDuration us
201 MEDIA_LOG_E("Seek to timestamp=" PUBLIC_LOG_D64 " failed, max=" PUBLIC_LOG_D64, timeStamp, fileDuration);
202 return false;
203 }
204 // when timestemp out of stream duration, seek to end of stream
205 if (streamDuration > 0 && timeStamp > streamDuration) {
206 MEDIA_LOG_W("Out of stream, seek to " PUBLIC_LOG_D64, timeStamp);
207 timeStamp = streamDuration;
208 }
209 if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
210 MEDIA_LOG_D("Reset timeStamp by start time [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
211 timeStamp, timeStamp + startTime);
212 timeStamp += startTime;
213 }
214 return true;
215 }
216
ConvertFlagsToFFmpeg(AVStream * avStream,int64_t ffTime,SeekMode mode,int64_t seekTime)217 int ConvertFlagsToFFmpeg(AVStream *avStream, int64_t ffTime, SeekMode mode, int64_t seekTime)
218 {
219 FALSE_RETURN_V_MSG_E(avStream != nullptr && avStream->codecpar != nullptr, -1, "AVStream is nullptr");
220 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ffTime == 0) {
221 return AVSEEK_FLAG_FRAME;
222 }
223 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && seekTime != 0) {
224 int64_t streamDuration = avStream->duration;
225 if (streamDuration == AV_NOPTS_VALUE || streamDuration <= 0) {
226 streamDuration = GetStreamDuration(*avStream);
227 }
228 // When the seekTime is within the last 0.5s, still use BACKWARD mode to ensure consistent func behavior.
229 int64_t buffering = ConvertTimeToFFmpeg(500 * MS_TO_NS, avStream->time_base); // 0.5s
230 if (streamDuration > 0 && (streamDuration < buffering || ffTime >= streamDuration - buffering)) {
231 return AVSEEK_FLAG_BACKWARD;
232 }
233 return g_seekModeToFFmpegSeekFlags.at(mode);
234 }
235 if (avStream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO || seekTime == 0) {
236 return AVSEEK_FLAG_BACKWARD;
237 }
238 if (mode == SeekMode::SEEK_NEXT_SYNC || mode == SeekMode::SEEK_PREVIOUS_SYNC) {
239 return g_seekModeToFFmpegSeekFlags.at(mode);
240 }
241 // find closest time in next and prev
242 int keyFrameNext = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_FRAME);
243 FALSE_RETURN_V_MSG_E(keyFrameNext >= 0, AVSEEK_FLAG_BACKWARD, "Not next key frame");
244
245 int keyFramePrev = av_index_search_timestamp(avStream, ffTime, AVSEEK_FLAG_BACKWARD);
246 FALSE_RETURN_V_MSG_E(keyFramePrev >= 0, AVSEEK_FLAG_FRAME, "Not pre key frame");
247
248 int64_t ffTimePrev = CalculateTimeByFrameIndex(avStream, keyFramePrev);
249 int64_t ffTimeNext = CalculateTimeByFrameIndex(avStream, keyFrameNext);
250 MEDIA_LOG_D("FfTime=" PUBLIC_LOG_D64 ", ffTimePrev=" PUBLIC_LOG_D64 ", ffTimeNext=" PUBLIC_LOG_D64,
251 ffTime, ffTimePrev, ffTimeNext);
252 if (ffTimePrev == ffTimeNext || (ffTimeNext - ffTime < ffTime - ffTimePrev)) {
253 return AVSEEK_FLAG_FRAME;
254 } else {
255 return AVSEEK_FLAG_BACKWARD;
256 }
257 }
258
IsSupportedTrack(const AVStream & avStream)259 bool IsSupportedTrack(const AVStream& avStream)
260 {
261 FALSE_RETURN_V_MSG_E(avStream.codecpar != nullptr, false, "Codecpar is nullptr");
262 if (std::find(g_streamMediaTypeVec.cbegin(), g_streamMediaTypeVec.cend(),
263 avStream.codecpar->codec_type) == g_streamMediaTypeVec.cend()) {
264 MEDIA_LOG_E("Unsupport track type: " PUBLIC_LOG_S,
265 ConvertFFmpegMediaTypeToString(avStream.codecpar->codec_type).data());
266 return false;
267 }
268 if (avStream.codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
269 if (avStream.codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
270 MEDIA_LOG_E("Unsupport raw video track");
271 return false;
272 }
273 if (std::count(g_imageCodecID.begin(), g_imageCodecID.end(), avStream.codecpar->codec_id) > 0) {
274 MEDIA_LOG_E("Unsupport image track");
275 return false;
276 }
277 }
278 return true;
279 }
280 } // namespace
281
282 std::atomic<int> FFmpegDemuxerPlugin::readatIndex_ = 0;
FFmpegDemuxerPlugin(std::string name)283 FFmpegDemuxerPlugin::FFmpegDemuxerPlugin(std::string name)
284 : DemuxerPlugin(std::move(name)),
285 seekable_(Seekable::SEEKABLE),
286 ioContext_(),
287 selectedTrackIds_(),
288 cacheQueue_("cacheQueue"),
289 streamParserInited_(false),
290 parserRefIoContext_()
291 {
292 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
293 MEDIA_LOG_D("In");
294 #ifndef _WIN32
295 (void)mallopt(M_SET_THREAD_CACHE, M_THREAD_CACHE_DISABLE);
296 (void)mallopt(M_DELAYED_FREE, M_DELAYED_FREE_DISABLE);
297 #endif
298 av_log_set_callback(FfmpegLogPrint);
299 #ifdef BUILD_ENG_VERSION
300 std::string dumpModeStr = OHOS::system::GetParameter("FFmpegDemuxerPlugin.dump", "0");
301 dumpMode_ = static_cast<DumpMode>(strtoul(dumpModeStr.c_str(), nullptr, 2)); // 2 is binary
302 MEDIA_LOG_D("Dump mode = %s(%lu)", dumpModeStr.c_str(), dumpMode_);
303 #endif
304 MEDIA_LOG_D("Out");
305 }
306
~FFmpegDemuxerPlugin()307 FFmpegDemuxerPlugin::~FFmpegDemuxerPlugin()
308 {
309 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
310 MEDIA_LOG_D("In");
311 #ifndef _WIN32
312 (void)mallopt(M_FLUSH_THREAD_CACHE, 0);
313 #endif
314 formatContext_ = nullptr;
315 pluginImpl_ = nullptr;
316 avbsfContext_ = nullptr;
317 streamParser_ = nullptr;
318 referenceParser_ = nullptr;
319 parserRefFormatContext_ = nullptr;
320 selectedTrackIds_.clear();
321 if (firstFrame_ != nullptr) {
322 av_packet_free(&firstFrame_);
323 av_free(firstFrame_);
324 firstFrame_ = nullptr;
325 }
326 MEDIA_LOG_D("Out");
327 }
328
Dump(const DumpParam & dumpParam)329 void FFmpegDemuxerPlugin::Dump(const DumpParam &dumpParam)
330 {
331 std::string path;
332 switch (dumpParam.mode) {
333 case DUMP_READAT_INPUT:
334 path = "Readat_index." + std::to_string(dumpParam.index) + "_offset." + std::to_string(dumpParam.offset) +
335 "_size." + std::to_string(dumpParam.size);
336 break;
337 case DUMP_AVPACKET_OUTPUT:
338 path = "AVPacket_index." + std::to_string(dumpParam.index) + "_track." +
339 std::to_string(dumpParam.trackId) + "_pts." + std::to_string(dumpParam.pts) + "_pos." +
340 std::to_string(dumpParam.pos);
341 break;
342 case DUMP_AVBUFFER_OUTPUT:
343 path = "AVBuffer_track." + std::to_string(dumpParam.trackId) + "_index." +
344 std::to_string(dumpParam.index) + "_pts." + std::to_string(dumpParam.pts);
345 break;
346 default:
347 return;
348 }
349 std::ofstream ofs;
350 path = "/data/ff_dump/" + path;
351 ofs.open(path, std::ios::out); // | std::ios::app
352 if (ofs.is_open()) {
353 ofs.write(reinterpret_cast<char*>(dumpParam.buf), dumpParam.size);
354 ofs.close();
355 }
356 MEDIA_LOG_D("Dump path:" PUBLIC_LOG_S, path.c_str());
357 }
358
Reset()359 Status FFmpegDemuxerPlugin::Reset()
360 {
361 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
362 MEDIA_LOG_D("In");
363 readatIndex_ = 0;
364 avpacketIndex_ = 0;
365 ioContext_.offset = 0;
366 ioContext_.eos = false;
367 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
368 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
369 }
370 selectedTrackIds_.clear();
371 pluginImpl_.reset();
372 formatContext_.reset();
373 avbsfContext_.reset();
374 trackMtx_.clear();
375 trackDfxInfoMap_.clear();
376 return Status::OK;
377 }
378
InitBitStreamContext(const AVStream & avStream)379 void FFmpegDemuxerPlugin::InitBitStreamContext(const AVStream& avStream)
380 {
381 FALSE_RETURN_MSG(avStream.codecpar != nullptr, "Codecpar is nullptr");
382 AVCodecID codecID = avStream.codecpar->codec_id;
383 MEDIA_LOG_D("For track " PUBLIC_LOG_D32, avStream.index);
384 FALSE_RETURN_MSG(g_bitstreamFilterMap.count(codecID) != 0, "Can not match any BitStreamContext");
385 const AVBitStreamFilter* avBitStreamFilter = av_bsf_get_by_name(g_bitstreamFilterMap.at(codecID).c_str());
386
387 FALSE_RETURN_MSG((avBitStreamFilter != nullptr), "Call av_bsf_get_by_name failed for" PUBLIC_LOG_S,
388 g_bitstreamFilterMap.at(codecID).c_str());
389
390 if (!avbsfContext_) {
391 AVBSFContext* avbsfContext {nullptr};
392 int ret = av_bsf_alloc(avBitStreamFilter, &avbsfContext);
393 FALSE_RETURN_MSG((ret >= 0 && avbsfContext != nullptr),
394 "Call av_bsf_alloc failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
395
396 ret = avcodec_parameters_copy(avbsfContext->par_in, avStream.codecpar);
397 FALSE_RETURN_MSG((ret >= 0), "Call avcodec_parameters_copy failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
398
399 ret = av_bsf_init(avbsfContext);
400 FALSE_RETURN_MSG((ret >= 0), "Call av_bsf_init failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
401
402 avbsfContext_ = std::shared_ptr<AVBSFContext>(avbsfContext, [](AVBSFContext* ptr) {
403 if (ptr) {
404 av_bsf_free(&ptr);
405 }
406 });
407 }
408 FALSE_RETURN_MSG(avbsfContext_ != nullptr, "Stream " PUBLIC_LOG_S " will not be converted to annexb",
409 g_bitstreamFilterMap.at(codecID).c_str());
410 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will convert to annexb", avStream.index);
411 }
412
ConvertAvcToAnnexb(AVPacket & pkt)413 Status FFmpegDemuxerPlugin::ConvertAvcToAnnexb(AVPacket& pkt)
414 {
415 int ret = av_bsf_send_packet(avbsfContext_.get(), &pkt);
416 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
417 "Call av_bsf_send_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
418 av_packet_unref(&pkt);
419
420 ret = av_bsf_receive_packet(avbsfContext_.get(), &pkt);
421 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
422 "Call av_bsf_receive_packet failed, err:" PUBLIC_LOG_S, AVStrError(ret).c_str());
423 return Status::OK;
424 }
425
ConvertHevcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)426 Status FFmpegDemuxerPlugin::ConvertHevcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
427 {
428 size_t cencInfoSize = 0;
429 uint8_t *cencInfo = av_packet_get_side_data(samplePacket->pkts[0], AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
430 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, cencInfo, cencInfoSize, false);
431 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) &&
432 streamParser_->IsSyncFrame(pkt.data, pkt.size)) {
433 pkt.flags = static_cast<int32_t>(static_cast<uint32_t>(pkt.flags) | static_cast<uint32_t>(AV_PKT_FLAG_KEY));
434 }
435 return Status::OK;
436 }
437
ConvertVvcToAnnexb(AVPacket & pkt,std::shared_ptr<SamplePacket> samplePacket)438 Status FFmpegDemuxerPlugin::ConvertVvcToAnnexb(AVPacket& pkt, std::shared_ptr<SamplePacket> samplePacket)
439 {
440 streamParser_->ConvertPacketToAnnexb(&(pkt.data), pkt.size, nullptr, 0, false);
441 return Status::OK;
442 }
443
WriteBuffer(std::shared_ptr<AVBuffer> outBuffer,const uint8_t * writeData,int32_t writeSize)444 Status FFmpegDemuxerPlugin::WriteBuffer(
445 std::shared_ptr<AVBuffer> outBuffer, const uint8_t *writeData, int32_t writeSize)
446 {
447 FALSE_RETURN_V_MSG_E(outBuffer != nullptr, Status::ERROR_NULL_POINTER, "Buffer is nullptr");
448 if (writeData != nullptr && writeSize > 0) {
449 FALSE_RETURN_V_MSG_E(outBuffer->memory_ != nullptr, Status::ERROR_NULL_POINTER, "Memory is nullptr");
450 int32_t ret = outBuffer->memory_->Write(writeData, writeSize, 0);
451 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_INVALID_OPERATION, "Memory write failed");
452 }
453
454 MEDIA_LOG_D("CurrentBuffer: [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_U32 "]",
455 outBuffer->pts_, outBuffer->duration_, outBuffer->flag_);
456 return Status::OK;
457 }
458
SetDrmCencInfo(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)459 Status FFmpegDemuxerPlugin::SetDrmCencInfo(
460 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
461 {
462 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
463 "Sample is nullptr");
464 FALSE_RETURN_V_MSG_E((samplePacket != nullptr && samplePacket->pkts.size() > 0), Status::ERROR_INVALID_OPERATION,
465 "Packet is nullptr");
466 FALSE_RETURN_V_MSG_E((samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0),
467 Status::ERROR_INVALID_OPERATION, "Packet empty");
468
469 size_t cencInfoSize = 0;
470 MetaDrmCencInfo *cencInfo = (MetaDrmCencInfo *)av_packet_get_side_data(samplePacket->pkts[0],
471 AV_PKT_DATA_ENCRYPTION_INFO, &cencInfoSize);
472 if ((cencInfo != nullptr) && (cencInfoSize != 0)) {
473 std::vector<uint8_t> drmCencVec(reinterpret_cast<uint8_t *>(cencInfo),
474 (reinterpret_cast<uint8_t *>(cencInfo)) + sizeof(MetaDrmCencInfo));
475 sample->meta_->SetData(Media::Tag::DRM_CENC_INFO, std::move(drmCencVec));
476 }
477 return Status::OK;
478 }
479
GetNextFrame(const uint8_t * data,const uint32_t size)480 bool FFmpegDemuxerPlugin::GetNextFrame(const uint8_t *data, const uint32_t size)
481 {
482 if (size < NAL_START_CODE_SIZE) {
483 return false;
484 }
485 bool hasShortStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 1); // 001
486 bool hasLongStartCode = (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1); // 0001
487 return hasShortStartCode || hasLongStartCode;
488 }
489
NeedCombineFrame(uint32_t trackId)490 bool FFmpegDemuxerPlugin::NeedCombineFrame(uint32_t trackId)
491 {
492 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, false, "AVFormatContext is nullptr");
493 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MPEGTS &&
494 formatContext_->streams[trackId]->codecpar->codec_id == AV_CODEC_ID_HEVC) {
495 return true;
496 }
497 return false;
498 }
499
CombinePackets(std::shared_ptr<SamplePacket> samplePacket)500 AVPacket* FFmpegDemuxerPlugin::CombinePackets(std::shared_ptr<SamplePacket> samplePacket)
501 {
502 AVPacket *tempPkt = nullptr;
503 if (NeedCombineFrame(samplePacket->pkts[0]->stream_index) && samplePacket->pkts.size() > 1) {
504 int totalSize = 0;
505 for (auto pkt : samplePacket->pkts) {
506 FALSE_RETURN_V_MSG_E(pkt != nullptr, nullptr, "AVPacket is nullptr");
507 totalSize += pkt->size;
508 }
509 tempPkt = av_packet_alloc();
510 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, nullptr, "Temp packet is nullptr");
511 int ret = av_new_packet(tempPkt, totalSize);
512 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "Call av_new_packet failed");
513 av_packet_copy_props(tempPkt, samplePacket->pkts[0]);
514 int offset = 0;
515 bool copySuccess = true;
516 for (auto pkt : samplePacket->pkts) {
517 if (pkt == nullptr) {
518 copySuccess = false;
519 MEDIA_LOG_E("Cache packet is nullptr");
520 break;
521 }
522 ret = memcpy_s(tempPkt->data + offset, pkt->size, pkt->data, pkt->size);
523 if (ret != EOK) {
524 copySuccess = false;
525 MEDIA_LOG_E("Memcpy failed, ret:" PUBLIC_LOG_D32, ret);
526 break;
527 }
528 offset += pkt->size;
529 }
530 if (!copySuccess) {
531 av_packet_free(&tempPkt);
532 av_free(tempPkt);
533 tempPkt = nullptr;
534 return nullptr;
535 }
536 tempPkt->size = totalSize;
537 MEDIA_LOG_D("Combine " PUBLIC_LOG_ZU " packets, total size=" PUBLIC_LOG_D32,
538 samplePacket->pkts.size(), totalSize);
539 } else {
540 tempPkt = samplePacket->pkts[0];
541 }
542 return tempPkt;
543 }
544
ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample,AVPacket * srcAVPacket,std::shared_ptr<SamplePacket> dstSamplePacket)545 Status FFmpegDemuxerPlugin::ConvertPacketToAnnexb(std::shared_ptr<AVBuffer> sample, AVPacket* srcAVPacket,
546 std::shared_ptr<SamplePacket> dstSamplePacket)
547 {
548 Status ret = Status::OK;
549 auto codecId = formatContext_->streams[srcAVPacket->stream_index]->codecpar->codec_id;
550 if (codecId == AV_CODEC_ID_HEVC && streamParser_ != nullptr && streamParserInited_) {
551 ret = ConvertHevcToAnnexb(*srcAVPacket, dstSamplePacket);
552 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_HEVC);
553 } else if (codecId == AV_CODEC_ID_VVC && streamParser_ != nullptr && streamParserInited_) {
554 ret = ConvertVvcToAnnexb(*srcAVPacket, dstSamplePacket);
555 } else if (codecId == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
556 ret = ConvertAvcToAnnexb(*srcAVPacket);
557 SetDropTag(*srcAVPacket, sample, AV_CODEC_ID_H264);
558 }
559 if (ioContext_.retry) {
560 ioContext_.retry = false;
561 formatContext_->pb->eof_reached = 0;
562 formatContext_->pb->error = 0;
563 cacheQueue_.Pop(dstSamplePacket->pkts[0]->stream_index);
564 return Status::ERROR_AGAIN;
565 }
566 return ret;
567 }
568
WriteBufferAttr(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)569 void FFmpegDemuxerPlugin::WriteBufferAttr(std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
570 {
571 AVStream *avStream = formatContext_->streams[samplePacket->pkts[0]->stream_index];
572 if (samplePacket->pkts[0]->pts != AV_NOPTS_VALUE) {
573 sample->pts_ = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->pts, avStream->time_base));
574 }
575 // durantion dts
576 if (samplePacket->pkts[0]->duration != AV_NOPTS_VALUE) {
577 int64_t duration = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->duration, avStream->time_base));
578 sample->duration_ = duration;
579 sample->meta_->SetData(Media::Tag::BUFFER_DURATION, duration);
580 }
581 if (samplePacket->pkts[0]->dts != AV_NOPTS_VALUE) {
582 int64_t dts = AvTime2Us(ConvertTimeFromFFmpeg(samplePacket->pkts[0]->dts, avStream->time_base));
583 sample->dts_ = dts;
584 sample->meta_->SetData(Media::Tag::BUFFER_DECODING_TIMESTAMP, dts);
585 }
586 if (avStream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
587 avStream->codecpar->codec_id != AV_CODEC_ID_H264 &&
588 firstFrame_ && samplePacket->pkts[0]->dts == firstFrame_->dts) {
589 if (streamParser_ != nullptr) {
590 streamParser_->ResetXPSSendStatus();
591 }
592 }
593 }
594
ConvertAVPacketToSample(std::shared_ptr<AVBuffer> sample,std::shared_ptr<SamplePacket> samplePacket)595 Status FFmpegDemuxerPlugin::ConvertAVPacketToSample(
596 std::shared_ptr<AVBuffer> sample, std::shared_ptr<SamplePacket> samplePacket)
597 {
598 FALSE_RETURN_V_MSG_E(samplePacket != nullptr && samplePacket->pkts.size() > 0 &&
599 samplePacket->pkts[0] != nullptr && samplePacket->pkts[0]->size >= 0,
600 Status::ERROR_INVALID_OPERATION, "Input packet is nullptr or empty");
601 MEDIA_LOG_D("Convert packet info for track " PUBLIC_LOG_D32, samplePacket->pkts[0]->stream_index);
602 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_ != nullptr, Status::ERROR_INVALID_OPERATION,
603 "Input sample is nullptr");
604
605 WriteBufferAttr(sample, samplePacket);
606
607 // convert
608 AVPacket *tempPkt = CombinePackets(samplePacket);
609 FALSE_RETURN_V_MSG_E(tempPkt != nullptr, Status::ERROR_INVALID_OPERATION, "Temp packet is empty");
610 Status ret = ConvertPacketToAnnexb(sample, tempPkt, samplePacket);
611 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Convert annexb failed");
612
613 // flag\copy
614 int32_t remainSize = tempPkt->size - static_cast<int32_t>(samplePacket->offset);
615 int32_t copySize = remainSize < sample->memory_->GetCapacity() ? remainSize : sample->memory_->GetCapacity();
616 MEDIA_LOG_D("Convert size [" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
617 tempPkt->size, remainSize, copySize, samplePacket->offset);
618 uint32_t flag = ConvertFlagsFromFFmpeg(*tempPkt, (copySize != tempPkt->size));
619 SetDrmCencInfo(sample, samplePacket);
620
621 sample->flag_ = flag;
622 ret = WriteBuffer(sample, tempPkt->data + samplePacket->offset, copySize);
623 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
624
625 if (!samplePacket->isEOS) {
626 trackDfxInfoMap_[tempPkt->stream_index].lastPts = sample->pts_;
627 trackDfxInfoMap_[tempPkt->stream_index].lastDurantion = sample->duration_;
628 trackDfxInfoMap_[tempPkt->stream_index].lastPos = tempPkt->pos;
629 }
630 #ifdef BUILD_ENG_VERSION
631 DumpParam dumpParam {DumpMode(DUMP_AVBUFFER_OUTPUT & dumpMode_), tempPkt->data + samplePacket->offset,
632 tempPkt->stream_index, -1, copySize, trackDfxInfoMap_[tempPkt->stream_index].frameIndex++, tempPkt->pts, -1};
633 Dump(dumpParam);
634 #endif
635 if (tempPkt != nullptr && tempPkt->size != samplePacket->pkts[0]->size) {
636 av_packet_free(&tempPkt);
637 av_free(tempPkt);
638 tempPkt = nullptr;
639 }
640
641 if (copySize < remainSize) {
642 samplePacket->offset += static_cast<uint32_t>(copySize);
643 MEDIA_LOG_D("Buffer is not enough, next buffer to copy remain data");
644 return Status::ERROR_NOT_ENOUGH_DATA;
645 }
646 return Status::OK;
647 }
648
PushEOSToAllCache()649 Status FFmpegDemuxerPlugin::PushEOSToAllCache()
650 {
651 Status ret = Status::OK;
652 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
653 auto streamIndex = selectedTrackIds_[i];
654 MEDIA_LOG_I("Track " PUBLIC_LOG_D32, streamIndex);
655 std::shared_ptr<SamplePacket> eosSample = std::make_shared<SamplePacket>();
656 eosSample->isEOS = true;
657 cacheQueue_.Push(streamIndex, eosSample);
658 ret = CheckCacheDataLimit(streamIndex);
659 }
660 return ret;
661 }
662
WebvttPktProcess(AVPacket * pkt)663 bool FFmpegDemuxerPlugin::WebvttPktProcess(AVPacket *pkt)
664 {
665 auto trackId = pkt->stream_index;
666 if (pkt->size > 0) { // vttc
667 return false;
668 } else { // vtte
669 if (cacheQueue_.HasCache(trackId)) {
670 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
671 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts.size() > 0 &&
672 cacheSamplePacket->pkts[0] != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
673 cacheSamplePacket->pkts[0]->duration = pkt->pts - cacheSamplePacket->pkts[0]->pts;
674 }
675 }
676 }
677 av_packet_free(&pkt);
678 return true;
679 }
680
IsWebvttMP4(const AVStream * avStream)681 bool FFmpegDemuxerPlugin::IsWebvttMP4(const AVStream *avStream)
682 {
683 if (avStream->codecpar->codec_id == AV_CODEC_ID_WEBVTT &&
684 FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4) {
685 return true;
686 }
687 return false;
688 }
689
WebvttMP4EOSProcess(const AVPacket * pkt)690 void FFmpegDemuxerPlugin::WebvttMP4EOSProcess(const AVPacket *pkt)
691 {
692 if (pkt != nullptr) {
693 auto trackId = pkt->stream_index;
694 AVStream *avStream = formatContext_->streams[trackId];
695 if (IsWebvttMP4(avStream) && pkt->size == 0 && cacheQueue_.HasCache(trackId)) {
696 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
697 if (cacheSamplePacket != nullptr && cacheSamplePacket->pkts[0]->duration == 0) {
698 cacheSamplePacket->pkts[0]->duration =
699 formatContext_->streams[pkt->stream_index]->duration - cacheSamplePacket->pkts[0]->pts;
700 }
701 }
702 }
703 }
704
ReadPacketToCacheQueue(const uint32_t readId)705 Status FFmpegDemuxerPlugin::ReadPacketToCacheQueue(const uint32_t readId)
706 {
707 std::lock_guard<std::mutex> lock(mutex_);
708 AVPacket *pkt = nullptr;
709 bool continueRead = true;
710 Status ret = Status::OK;
711 while (continueRead) {
712 if (pkt == nullptr) {
713 pkt = av_packet_alloc();
714 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
715 }
716 std::unique_lock<std::mutex> sLock(syncMutex_);
717 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
718 sLock.unlock();
719 if (ffmpegRet == AVERROR_EOF) { // eos
720 WebvttMP4EOSProcess(pkt);
721 av_packet_free(&pkt);
722 ret = PushEOSToAllCache();
723 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Push eos failed");
724 return Status::END_OF_STREAM;
725 }
726 if (ffmpegRet < 0) { // fail
727 av_packet_free(&pkt);
728 MEDIA_LOG_E("Call av_read_frame failed:" PUBLIC_LOG_S ", retry: " PUBLIC_LOG_D32,
729 AVStrError(ffmpegRet).c_str(), int(ioContext_.retry));
730 if (ioContext_.retry) {
731 formatContext_->pb->eof_reached = 0;
732 formatContext_->pb->error = 0;
733 ioContext_.retry = false;
734 return Status::ERROR_AGAIN;
735 }
736 return Status::ERROR_UNKNOWN;
737 }
738 auto trackId = pkt->stream_index;
739 if (!TrackIsSelected(trackId)) {
740 av_packet_unref(pkt);
741 continue;
742 }
743 AVStream *avStream = formatContext_->streams[trackId];
744 if (IsWebvttMP4(avStream) && WebvttPktProcess(pkt)) {
745 break;
746 } else if (!IsWebvttMP4(avStream) && (!NeedCombineFrame(readId) ||
747 (cacheQueue_.HasCache(static_cast<uint32_t>(trackId)) && GetNextFrame(pkt->data, pkt->size)))) {
748 continueRead = false;
749 }
750 ret = AddPacketToCacheQueue(pkt);
751 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Add cache failed");
752 pkt = nullptr;
753 }
754 return ret;
755 }
756
SetEosSample(std::shared_ptr<AVBuffer> sample)757 Status FFmpegDemuxerPlugin::SetEosSample(std::shared_ptr<AVBuffer> sample)
758 {
759 MEDIA_LOG_D("In");
760 sample->pts_ = 0;
761 sample->flag_ = (uint32_t)(AVBufferFlag::EOS);
762 Status ret = WriteBuffer(sample, nullptr, 0);
763 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Write buffer failed");
764 MEDIA_LOG_I("Out");
765 return Status::OK;
766 }
767
Start()768 Status FFmpegDemuxerPlugin::Start()
769 {
770 return Status::OK;
771 }
772
Stop()773 Status FFmpegDemuxerPlugin::Stop()
774 {
775 return Status::OK;
776 }
777
778 // Write packet unimplemented, return 0
AVWritePacket(void * opaque,uint8_t * buf,int bufSize)779 int FFmpegDemuxerPlugin::AVWritePacket(void* opaque, uint8_t* buf, int bufSize)
780 {
781 (void)opaque;
782 (void)buf;
783 (void)bufSize;
784 return 0;
785 }
786
CheckContextIsValid(void * opaque,int & bufSize)787 int FFmpegDemuxerPlugin::CheckContextIsValid(void* opaque, int &bufSize)
788 {
789 int ret = -1;
790 auto ioContext = static_cast<IOContext*>(opaque);
791 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
792 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, ret, "DataSource is nullptr");
793 FALSE_RETURN_V_MSG_E(ioContext->offset <= INT64_MAX - static_cast<int64_t>(bufSize), ret, "Offset invalid");
794
795 if (ioContext->dataSource->IsDash() && ioContext->eos == true) {
796 MEDIA_LOG_I("Read eos");
797 return AVERROR_EOF;
798 }
799
800 MEDIA_LOG_D("Offset: " PUBLIC_LOG_D64 ", totalSize: " PUBLIC_LOG_U64, ioContext->offset, ioContext->fileSize);
801 if (ioContext->fileSize > 0) {
802 FALSE_RETURN_V_MSG_E(static_cast<uint64_t>(ioContext->offset) <= ioContext->fileSize, ret, "Out of file size");
803 if (static_cast<size_t>(ioContext->offset + bufSize) > ioContext->fileSize) {
804 bufSize = static_cast<int64_t>(ioContext->fileSize) - ioContext->offset;
805 }
806 }
807 return 0;
808 }
809
810 // Write packet data into the buffer provided by ffmpeg
AVReadPacket(void * opaque,uint8_t * buf,int bufSize)811 int FFmpegDemuxerPlugin::AVReadPacket(void* opaque, uint8_t* buf, int bufSize)
812 {
813 int ret = CheckContextIsValid(opaque, bufSize);
814 FALSE_RETURN_V(ret == 0, ret);
815
816 ret = -1;
817 auto ioContext = static_cast<IOContext*>(opaque);
818 FALSE_RETURN_V_MSG_E(ioContext != nullptr, ret, "IOContext is nullptr");
819 auto buffer = std::make_shared<Buffer>();
820 FALSE_RETURN_V_MSG_E(buffer != nullptr, ret, "Buffer is nullptr");
821 auto bufData = buffer->WrapMemory(buf, bufSize, 0);
822 FALSE_RETURN_V_MSG_E(buffer->GetMemory() != nullptr, ret, "Memory is nullptr");
823
824 MediaAVCodec::AVCodecTrace trace("AVReadPacket_ReadAt");
825 auto result = ioContext->dataSource->ReadAt(ioContext->offset, buffer, static_cast<size_t>(bufSize));
826 int dataSize = static_cast<int>(buffer->GetMemory()->GetSize());
827 MEDIA_LOG_D("Want:" PUBLIC_LOG_D32 ", Get:" PUBLIC_LOG_D32 ", offset:" PUBLIC_LOG_D64 ", index:" PUBLIC_LOG_D32,
828 bufSize, dataSize, ioContext->offset, readatIndex_.load());
829 #ifdef BUILD_ENG_VERSION
830 DumpParam dumpParam {DumpMode(DUMP_READAT_INPUT & ioContext->dumpMode), buf, -1, ioContext->offset,
831 dataSize, readatIndex_++, -1, -1};
832 Dump(dumpParam);
833 #endif
834 switch (result) {
835 case Status::OK:
836 case Status::ERROR_AGAIN:
837 if (dataSize == 0) {
838 MEDIA_LOG_I("Read again");
839 ioContext->retry = true;
840 } else {
841 ioContext->offset += dataSize;
842 ret = dataSize;
843 }
844 break;
845 case Status::END_OF_STREAM:
846 MEDIA_LOG_I("Read end");
847 ioContext->eos = true;
848 ret = AVERROR_EOF;
849 break;
850 default:
851 MEDIA_LOG_I("Read failed " PUBLIC_LOG_D32, static_cast<int>(result));
852 break;
853 }
854
855 if (!ioContext->initCompleted) {
856 if (ioContext->initDownloadDataSize <= UINT32_MAX - static_cast<uint32_t>(dataSize)) {
857 ioContext->initDownloadDataSize += static_cast<uint32_t>(dataSize);
858 } else {
859 MEDIA_LOG_W("DataSize " PUBLIC_LOG_U32 " is invalid", static_cast<uint32_t>(dataSize));
860 }
861 }
862
863 return ret;
864 }
865
AVSeek(void * opaque,int64_t offset,int whence)866 int64_t FFmpegDemuxerPlugin::AVSeek(void* opaque, int64_t offset, int whence)
867 {
868 auto ioContext = static_cast<IOContext*>(opaque);
869 uint64_t newPos = 0;
870 FALSE_RETURN_V_MSG_E(ioContext != nullptr, newPos, "IOContext is nullptr");
871 switch (whence) {
872 case SEEK_SET:
873 newPos = static_cast<uint64_t>(offset);
874 ioContext->offset = newPos;
875 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
876 whence, offset, newPos);
877 break;
878 case SEEK_CUR:
879 newPos = ioContext->offset + offset;
880 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
881 whence, offset, newPos);
882 break;
883 case SEEK_END:
884 case AVSEEK_SIZE: {
885 FALSE_RETURN_V_MSG_E(ioContext->dataSource != nullptr, newPos, "DataSource is nullptr");
886 if (ioContext->dataSource->IsDash()) {
887 return -1;
888 }
889 uint64_t mediaDataSize = 0;
890 if (ioContext->dataSource->GetSize(mediaDataSize) == Status::OK && (mediaDataSize > 0)) {
891 newPos = mediaDataSize + offset;
892 MEDIA_LOG_D("Whence: " PUBLIC_LOG_D32 ", pos = " PUBLIC_LOG_D64 ", newPos = " PUBLIC_LOG_U64,
893 whence, offset, newPos);
894 }
895 break;
896 }
897 default:
898 MEDIA_LOG_E("Unexpected whence " PUBLIC_LOG_D32, whence);
899 break;
900 }
901 if (whence != AVSEEK_SIZE) {
902 ioContext->offset = newPos;
903 }
904 MEDIA_LOG_D("Current offset: " PUBLIC_LOG_D64 ", new pos: " PUBLIC_LOG_U64, ioContext->offset, newPos);
905 return newPos;
906 }
907
AllocAVIOContext(int flags,IOContext * ioContext)908 AVIOContext* FFmpegDemuxerPlugin::AllocAVIOContext(int flags, IOContext *ioContext)
909 {
910 auto buffer = static_cast<unsigned char*>(av_malloc(DEFAULT_READ_SIZE));
911 FALSE_RETURN_V_MSG_E(buffer != nullptr, nullptr, "Call av_malloc failed");
912
913 AVIOContext* avioContext = avio_alloc_context(
914 buffer, DEFAULT_READ_SIZE, flags & AVIO_FLAG_WRITE, static_cast<void*>(ioContext),
915 AVReadPacket, AVWritePacket, AVSeek);
916 if (avioContext == nullptr) {
917 MEDIA_LOG_E("Call avio_alloc_context failed");
918 av_free(buffer);
919 return nullptr;
920 }
921 avioContext->seekable = (seekable_ == Seekable::SEEKABLE) ? AVIO_SEEKABLE_NORMAL : 0;
922 if (!(static_cast<uint32_t>(flags) & static_cast<uint32_t>(AVIO_FLAG_WRITE))) {
923 avioContext->buf_ptr = avioContext->buf_end;
924 avioContext->write_flag = 0;
925 }
926 return avioContext;
927 }
928
FreeContext(AVFormatContext * formatContext,AVIOContext * avioContext)929 void FreeContext(AVFormatContext* formatContext, AVIOContext* avioContext)
930 {
931 if (formatContext) {
932 avformat_close_input(&formatContext);
933 }
934 if (avioContext) {
935 if (avioContext->buffer) {
936 av_freep(&(avioContext->buffer));
937 }
938 avio_context_free(&avioContext);
939 }
940 }
941
ParseHeader(AVFormatContext * formatContext,std::shared_ptr<AVInputFormat> pluginImpl,AVDictionary ** options)942 int32_t ParseHeader(AVFormatContext* formatContext, std::shared_ptr<AVInputFormat> pluginImpl, AVDictionary **options)
943 {
944 FALSE_RETURN_V_MSG_E(formatContext && pluginImpl, -1, "AVFormatContext is nullptr");
945 MediaAVCodec::AVCodecTrace trace("ffmpeg_init");
946
947 AVIOContext* avioContext = formatContext->pb;
948 auto begin = std::chrono::steady_clock::now();
949 int ret = avformat_open_input(&formatContext, nullptr, pluginImpl.get(), options);
950 if (ret < 0) {
951 FreeContext(formatContext, avioContext);
952 MEDIA_LOG_E("Call avformat_open_input failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
953 pluginImpl->name, AVStrError(ret).c_str());
954 return ret;
955 }
956
957 auto open = std::chrono::steady_clock::now();
958 if (FFmpegFormatHelper::GetFileTypeByName(*formatContext) == FileType::FLV) { // Fix init live-flv-source too slow
959 formatContext->probesize = LIVE_FLV_PROBE_SIZE;
960 }
961
962 ret = avformat_find_stream_info(formatContext, NULL);
963 auto parse = std::chrono::steady_clock::now();
964 int64_t openSpend = static_cast<int64_t>(
965 std::chrono::duration_cast<std::chrono::milliseconds>(open - begin).count());
966 int64_t parseSpend = static_cast<int64_t>(
967 std::chrono::duration_cast<std::chrono::milliseconds>(parse - open).count());
968 if ((parseSpend < 0) || (openSpend > INT64_MAX - parseSpend) || (openSpend + parseSpend > INIT_TIME_THRESHOLD)) {
969 MEDIA_LOG_W("Spend [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]", openSpend, parseSpend);
970 }
971 if (ret < 0) {
972 FreeContext(formatContext, avioContext);
973 MEDIA_LOG_E("Parse stream info failed by " PUBLIC_LOG_S ", err:" PUBLIC_LOG_S,
974 pluginImpl->name, AVStrError(ret).c_str());
975 return ret;
976 }
977 return 0;
978 }
979
InitAVFormatContext(IOContext * ioContext)980 std::shared_ptr<AVFormatContext> FFmpegDemuxerPlugin::InitAVFormatContext(IOContext *ioContext)
981 {
982 AVFormatContext* formatContext = avformat_alloc_context();
983 FALSE_RETURN_V_MSG_E(formatContext != nullptr, nullptr, "AVFormatContext is nullptr");
984
985 formatContext->pb = AllocAVIOContext(AVIO_FLAG_READ, ioContext);
986 if (formatContext->pb == nullptr) {
987 FreeContext(formatContext, nullptr);
988 return nullptr;
989 }
990
991 formatContext->flags = static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_CUSTOM_IO);
992 if (std::string(pluginImpl_->name) == "mp3") {
993 formatContext->flags =
994 static_cast<uint32_t>(formatContext->flags) | static_cast<uint32_t>(AVFMT_FLAG_FAST_SEEK);
995 }
996 AVDictionary *options = nullptr;
997 if (ioContext_.dataSource->IsDash()) {
998 av_dict_set(&options, "use_tfdt", "true", 0);
999 }
1000
1001 int ret = ParseHeader(formatContext, pluginImpl_, &options);
1002 av_dict_free(&options);
1003 FALSE_RETURN_V_MSG_E(ret >= 0, nullptr, "ParseHeader failed");
1004
1005 std::shared_ptr<AVFormatContext> retFormatContext =
1006 std::shared_ptr<AVFormatContext>(formatContext, [](AVFormatContext *ptr) {
1007 if (ptr) {
1008 auto ctx = ptr->pb;
1009 avformat_close_input(&ptr);
1010 if (ctx) {
1011 ctx->opaque = nullptr;
1012 av_freep(&(ctx->buffer));
1013 av_opt_free(ctx);
1014 avio_context_free(&ctx);
1015 ctx = nullptr;
1016 }
1017 }
1018 });
1019 return retFormatContext;
1020 }
1021
NotifyInitializationCompleted()1022 void FFmpegDemuxerPlugin::NotifyInitializationCompleted()
1023 {
1024 ioContext_.initCompleted = true;
1025 if (ioContext_.initDownloadDataSize >= INIT_DOWNLOADS_DATA_SIZE_THRESHOLD) {
1026 MEDIA_LOG_I("Large init size %{public}u", ioContext_.initDownloadDataSize);
1027 }
1028 }
1029
SetDataSource(const std::shared_ptr<DataSource> & source)1030 Status FFmpegDemuxerPlugin::SetDataSource(const std::shared_ptr<DataSource>& source)
1031 {
1032 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1033 FALSE_RETURN_V_MSG_E(formatContext_ == nullptr, Status::ERROR_WRONG_STATE, "AVFormatContext is nullptr");
1034 FALSE_RETURN_V_MSG_E(source != nullptr, Status::ERROR_INVALID_PARAMETER, "DataSource is nullptr");
1035 ioContext_.dataSource = source;
1036 ioContext_.offset = 0;
1037 ioContext_.eos = false;
1038 ioContext_.dumpMode = dumpMode_;
1039 seekable_ = ioContext_.dataSource->IsDash() ? Plugins::Seekable::UNSEEKABLE : source->GetSeekable();
1040 if (seekable_ == Plugins::Seekable::SEEKABLE) {
1041 ioContext_.dataSource->GetSize(ioContext_.fileSize);
1042 } else {
1043 ioContext_.fileSize = -1;
1044 }
1045 MEDIA_LOG_I("FileSize: " PUBLIC_LOG_U64 ", seekable: " PUBLIC_LOG_D32, ioContext_.fileSize, seekable_);
1046 {
1047 std::lock_guard<std::mutex> glock(g_mtx);
1048 pluginImpl_ = g_pluginInputFormat[pluginName_];
1049 }
1050 FALSE_RETURN_V_MSG_E(pluginImpl_ != nullptr, Status::ERROR_UNSUPPORTED_FORMAT, "No match inputformat");
1051 formatContext_ = InitAVFormatContext(&ioContext_);
1052 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1053 InitParser();
1054
1055 NotifyInitializationCompleted();
1056 MEDIA_LOG_I("Out");
1057 cachelimitSize_ = DEFAULT_CACHE_LIMIT;
1058 return Status::OK;
1059 }
1060
InitParser()1061 void FFmpegDemuxerPlugin::InitParser()
1062 {
1063 FALSE_RETURN_MSG(formatContext_ != nullptr, "AVFormatContext is nullptr");
1064 ParserBoxInfo();
1065 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1066 if (g_bitstreamFilterMap.count(formatContext_->streams[trackIndex]->codecpar->codec_id) != 0) {
1067 InitBitStreamContext(*(formatContext_->streams[trackIndex]));
1068 break;
1069 }
1070 if (HaveValidParser(formatContext_->streams[trackIndex]->codecpar->codec_id)) {
1071 streamParser_ = StreamParserManager::Create(g_streamParserMap.at(
1072 formatContext_->streams[trackIndex]->codecpar->codec_id));
1073 if (streamParser_ == nullptr) {
1074 MEDIA_LOG_W("Init failed");
1075 } else {
1076 MEDIA_LOG_D("Track " PUBLIC_LOG_D32 " will be converted", trackIndex);
1077 }
1078 break;
1079 }
1080 }
1081 }
1082
GetSeiInfo()1083 Status FFmpegDemuxerPlugin::GetSeiInfo()
1084 {
1085 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1086 Status ret = Status::OK;
1087 if (streamParser_ != nullptr && !streamParserInited_) {
1088 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1089 auto avStream = formatContext_->streams[trackIndex];
1090 if (HaveValidParser(avStream->codecpar->codec_id)) {
1091 ret = GetVideoFirstKeyFrame(trackIndex);
1092 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, Status::ERROR_NO_MEMORY, "No memory");
1093 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr && firstFrame_->data != nullptr,
1094 Status::ERROR_WRONG_STATE, "Get first frame failed");
1095 bool convertRet = streamParser_->ConvertExtraDataToAnnexb(
1096 avStream->codecpar->extradata, avStream->codecpar->extradata_size);
1097 FALSE_RETURN_V_MSG_E(convertRet, Status::ERROR_INVALID_DATA, "ConvertExtraDataToAnnexb failed");
1098 streamParserInited_ = true;
1099 break;
1100 }
1101 }
1102 }
1103 return ret;
1104 }
1105
GetMediaInfo(MediaInfo & mediaInfo)1106 Status FFmpegDemuxerPlugin::GetMediaInfo(MediaInfo& mediaInfo)
1107 {
1108 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetMediaInfo");
1109 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1110 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1111
1112 Status ret = GetSeiInfo();
1113 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "GetSeiInfo failed");
1114
1115 FFmpegFormatHelper::ParseMediaInfo(*formatContext_, mediaInfo.general);
1116 DemuxerLogCompressor::StringifyMeta(mediaInfo.general, -1); // source meta
1117 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1118 Meta meta;
1119 auto avStream = formatContext_->streams[trackIndex];
1120 if (avStream == nullptr) {
1121 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " info is nullptr", trackIndex);
1122 mediaInfo.tracks.push_back(meta);
1123 continue;
1124 }
1125 FFmpegFormatHelper::ParseTrackInfo(*avStream, meta, *formatContext_);
1126 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC) {
1127 if (streamParser_ != nullptr && streamParserInited_ && firstFrame_ != nullptr) {
1128 streamParser_->ConvertPacketToAnnexb(&(firstFrame_->data), firstFrame_->size, nullptr, 0, false);
1129 streamParser_->ParseAnnexbExtraData(firstFrame_->data, firstFrame_->size);
1130 // Parser only sends xps info when first call ConvertPacketToAnnexb
1131 // readSample will call ConvertPacketToAnnexb again, so rest here
1132 streamParser_->ResetXPSSendStatus();
1133 ParseHEVCMetadataInfo(*avStream, meta);
1134 } else {
1135 MEDIA_LOG_W("Parse hevc info failed");
1136 }
1137 }
1138 if (avStream->codecpar->codec_id == AV_CODEC_ID_HEVC ||
1139 avStream->codecpar->codec_id == AV_CODEC_ID_H264 ||
1140 avStream->codecpar->codec_id == AV_CODEC_ID_VVC) {
1141 ConvertCsdToAnnexb(*avStream, meta);
1142 }
1143 mediaInfo.tracks.push_back(meta);
1144 DemuxerLogCompressor::StringifyMeta(meta, trackIndex);
1145 }
1146 return Status::OK;
1147 }
1148
GetUserMeta(std::shared_ptr<Meta> meta)1149 Status FFmpegDemuxerPlugin::GetUserMeta(std::shared_ptr<Meta> meta)
1150 {
1151 MediaAVCodec::AVCodecTrace trace("FFmpegDemuxerPlugin::GetUserMeta");
1152 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1153 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1154 FALSE_RETURN_V_MSG_E(meta != nullptr, Status::ERROR_NULL_POINTER, "Meta is nullptr");
1155
1156 FFmpegFormatHelper::ParseUserMeta(*formatContext_, meta);
1157 return Status::OK;
1158 }
1159
ParseDrmInfo(const MetaDrmInfo * const metaDrmInfo,size_t drmInfoSize,std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1160 void FFmpegDemuxerPlugin::ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, size_t drmInfoSize,
1161 std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1162 {
1163 MEDIA_LOG_D("In");
1164 size_t infoCount = drmInfoSize / sizeof(MetaDrmInfo);
1165 for (size_t index = 0; index < infoCount; index++) {
1166 std::stringstream ssConverter;
1167 std::string uuid;
1168 for (uint32_t i = 0; i < metaDrmInfo[index].uuidLen; i++) {
1169 int32_t singleUuid = static_cast<int32_t>(metaDrmInfo[index].uuid[i]);
1170 ssConverter << std::hex << std::setfill('0') << std::setw(2) << singleUuid; // 2:w
1171 uuid = ssConverter.str();
1172 }
1173 drmInfo.insert({ uuid, std::vector<uint8_t>(metaDrmInfo[index].pssh,
1174 metaDrmInfo[index].pssh + metaDrmInfo[index].psshLen) });
1175 }
1176 }
1177
GetDrmInfo(std::multimap<std::string,std::vector<uint8_t>> & drmInfo)1178 Status FFmpegDemuxerPlugin::GetDrmInfo(std::multimap<std::string, std::vector<uint8_t>>& drmInfo)
1179 {
1180 MEDIA_LOG_D("In");
1181 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1182 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1183
1184 for (uint32_t trackIndex = 0; trackIndex < formatContext_->nb_streams; ++trackIndex) {
1185 Meta meta;
1186 AVStream *avStream = formatContext_->streams[trackIndex];
1187 if (avStream == nullptr) {
1188 MEDIA_LOG_W("Track " PUBLIC_LOG_D32 " is nullptr", trackIndex);
1189 continue;
1190 }
1191 MEDIA_LOG_D("GetDrmInfo by stream side data");
1192 size_t drmInfoSize = 0;
1193 MetaDrmInfo *tmpDrmInfo = (MetaDrmInfo *)av_stream_get_side_data(avStream,
1194 AV_PKT_DATA_ENCRYPTION_INIT_INFO, &drmInfoSize);
1195 if (tmpDrmInfo != nullptr && drmInfoSize != 0) {
1196 ParseDrmInfo(tmpDrmInfo, drmInfoSize, drmInfo);
1197 }
1198 }
1199 return Status::OK;
1200 }
1201
ConvertCsdToAnnexb(const AVStream & avStream,Meta & format)1202 void FFmpegDemuxerPlugin::ConvertCsdToAnnexb(const AVStream& avStream, Meta &format)
1203 {
1204 uint8_t *extradata = avStream.codecpar->extradata;
1205 int32_t extradataSize = avStream.codecpar->extradata_size;
1206 if (HaveValidParser(avStream.codecpar->codec_id) && streamParser_ != nullptr && streamParserInited_) {
1207 streamParser_->ConvertPacketToAnnexb(&(extradata), extradataSize, nullptr, 0, true);
1208 } else if (avStream.codecpar->codec_id == AV_CODEC_ID_H264 && avbsfContext_ != nullptr) {
1209 if (avbsfContext_->par_out->extradata != nullptr && avbsfContext_->par_out->extradata_size > 0) {
1210 extradata = avbsfContext_->par_out->extradata;
1211 extradataSize = avbsfContext_->par_out->extradata_size;
1212 }
1213 }
1214 if (extradata != nullptr && extradataSize > 0) {
1215 std::vector<uint8_t> extra(extradataSize);
1216 extra.assign(extradata, extradata + extradataSize);
1217 format.Set<Tag::MEDIA_CODEC_CONFIG>(extra);
1218 }
1219 }
1220
AddPacketToCacheQueue(AVPacket * pkt)1221 Status FFmpegDemuxerPlugin::AddPacketToCacheQueue(AVPacket *pkt)
1222 {
1223 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Pkt is nullptr");
1224 #ifdef BUILD_ENG_VERSION
1225 DumpParam dumpParam {DumpMode(DUMP_AVPACKET_OUTPUT & dumpMode_), pkt->data, pkt->stream_index, -1, pkt->size,
1226 avpacketIndex_++, pkt->pts, pkt->pos};
1227 Dump(dumpParam);
1228 #endif
1229 auto trackId = pkt->stream_index;
1230 Status ret = Status::OK;
1231 if (NeedCombineFrame(trackId) && !GetNextFrame(pkt->data, pkt->size) && cacheQueue_.HasCache(trackId)) {
1232 std::shared_ptr<SamplePacket> cacheSamplePacket = cacheQueue_.Back(static_cast<uint32_t>(trackId));
1233 if (cacheSamplePacket != nullptr) {
1234 cacheSamplePacket->pkts.push_back(pkt);
1235 }
1236 } else {
1237 std::shared_ptr<SamplePacket> cacheSamplePacket = std::make_shared<SamplePacket>();
1238 if (cacheSamplePacket != nullptr) {
1239 cacheSamplePacket->pkts.push_back(pkt);
1240 cacheSamplePacket->offset = 0;
1241 cacheQueue_.Push(static_cast<uint32_t>(trackId), cacheSamplePacket);
1242 ret = CheckCacheDataLimit(static_cast<uint32_t>(trackId));
1243 }
1244 }
1245 return ret;
1246 }
1247
GetVideoFirstKeyFrame(uint32_t trackIndex)1248 Status FFmpegDemuxerPlugin::GetVideoFirstKeyFrame(uint32_t trackIndex)
1249 {
1250 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1251 AVPacket *pkt = nullptr;
1252 Status ret = Status::OK;
1253 while (1) {
1254 if (pkt == nullptr) {
1255 pkt = av_packet_alloc();
1256 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1257 }
1258
1259 std::unique_lock<std::mutex> sLock(syncMutex_);
1260 int ffmpegRet = av_read_frame(formatContext_.get(), pkt);
1261 sLock.unlock();
1262 if (ffmpegRet < 0) {
1263 MEDIA_LOG_E("Call av_read_frame failed, ret:" PUBLIC_LOG_D32, ffmpegRet);
1264 av_packet_unref(pkt);
1265 break;
1266 }
1267 cacheQueue_.AddTrackQueue(pkt->stream_index);
1268 ret = AddPacketToCacheQueue(pkt);
1269 if (ret != Status::OK) {
1270 return ret;
1271 }
1272
1273 if (static_cast<uint32_t>(pkt->stream_index) == trackIndex) {
1274 firstFrame_ = av_packet_alloc();
1275 FALSE_RETURN_V_MSG_E(firstFrame_ != nullptr, Status::ERROR_NULL_POINTER, "Call av_packet_alloc failed");
1276 int avRet = av_new_packet(firstFrame_, pkt->size);
1277 FALSE_RETURN_V_MSG_E(avRet >= 0, Status::ERROR_INVALID_DATA, "Call av_new_packet failed");
1278 av_packet_copy_props(firstFrame_, pkt);
1279 memcpy_s(firstFrame_->data, pkt->size, pkt->data, pkt->size);
1280 break;
1281 }
1282 pkt = nullptr;
1283 }
1284 return ret;
1285 }
1286
ParseHEVCMetadataInfo(const AVStream & avStream,Meta & format)1287 void FFmpegDemuxerPlugin::ParseHEVCMetadataInfo(const AVStream& avStream, Meta& format)
1288 {
1289 HevcParseFormat parse;
1290 parse.isHdrVivid = streamParser_->IsHdrVivid();
1291 parse.colorRange = streamParser_->GetColorRange();
1292 parse.colorPrimaries = streamParser_->GetColorPrimaries();
1293 parse.colorTransfer = streamParser_->GetColorTransfer();
1294 parse.colorMatrixCoeff = streamParser_->GetColorMatrixCoeff();
1295 parse.profile = streamParser_->GetProfileIdc();
1296 parse.level = streamParser_->GetLevelIdc();
1297 parse.chromaLocation = streamParser_->GetChromaLocation();
1298 parse.picWidInLumaSamples = streamParser_->GetPicWidInLumaSamples();
1299 parse.picHetInLumaSamples = streamParser_->GetPicHetInLumaSamples();
1300
1301 FFmpegFormatHelper::ParseHevcInfo(*formatContext_, parse, format);
1302 }
1303
TrackIsSelected(const uint32_t trackId)1304 bool FFmpegDemuxerPlugin::TrackIsSelected(const uint32_t trackId)
1305 {
1306 return std::any_of(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1307 [trackId](uint32_t id) { return id == trackId; });
1308 }
1309
SelectTrack(uint32_t trackId)1310 Status FFmpegDemuxerPlugin::SelectTrack(uint32_t trackId)
1311 {
1312 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1313 MEDIA_LOG_I("Select " PUBLIC_LOG_D32, trackId);
1314 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1315 if (trackId >= static_cast<uint32_t>(formatContext_.get()->nb_streams)) {
1316 MEDIA_LOG_E("Track is invalid, just have " PUBLIC_LOG_D32 " tracks in file", formatContext_.get()->nb_streams);
1317 return Status::ERROR_INVALID_PARAMETER;
1318 }
1319
1320 AVStream* avStream = formatContext_->streams[trackId];
1321 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1322 if (!IsSupportedTrack(*avStream)) {
1323 MEDIA_LOG_E("Track type is unsupport");
1324 return Status::ERROR_INVALID_PARAMETER;
1325 }
1326
1327 if (!TrackIsSelected(trackId)) {
1328 selectedTrackIds_.push_back(trackId);
1329 trackMtx_[trackId] = std::make_shared<std::mutex>();
1330 trackDfxInfoMap_[trackId] = {0, -1, -1};
1331 return cacheQueue_.AddTrackQueue(trackId);
1332 } else {
1333 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " has been selected", trackId);
1334 }
1335 return Status::OK;
1336 }
1337
UnselectTrack(uint32_t trackId)1338 Status FFmpegDemuxerPlugin::UnselectTrack(uint32_t trackId)
1339 {
1340 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1341 MEDIA_LOG_I("Unselect " PUBLIC_LOG_D32, trackId);
1342 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1343 auto index = std::find_if(selectedTrackIds_.begin(), selectedTrackIds_.end(),
1344 [trackId](uint32_t selectedId) {return trackId == selectedId; });
1345 if (TrackIsSelected(trackId)) {
1346 selectedTrackIds_.erase(index);
1347 trackMtx_.erase(trackId);
1348 trackDfxInfoMap_.erase(trackId);
1349 return cacheQueue_.RemoveTrackQueue(trackId);
1350 } else {
1351 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " is not in selected list", trackId);
1352 }
1353 return Status::OK;
1354 }
1355
SeekTo(int32_t trackId,int64_t seekTime,SeekMode mode,int64_t & realSeekTime)1356 Status FFmpegDemuxerPlugin::SeekTo(int32_t trackId, int64_t seekTime, SeekMode mode, int64_t& realSeekTime)
1357 {
1358 (void) trackId;
1359 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1360 MediaAVCodec::AVCodecTrace trace("SeekTo");
1361 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1362 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1363
1364 FALSE_RETURN_V_MSG_E(seekTime >= 0, Status::ERROR_INVALID_PARAMETER,
1365 "Seek time " PUBLIC_LOG_D64 " is not unsupported", seekTime);
1366 FALSE_RETURN_V_MSG_E(g_seekModeToFFmpegSeekFlags.count(mode) != 0, Status::ERROR_INVALID_PARAMETER,
1367 "Seek mode " PUBLIC_LOG_D32 " is not unsupported", static_cast<uint32_t>(mode));
1368
1369 int trackIndex = static_cast<int>(selectedTrackIds_[0]);
1370 for (size_t i = 1; i < selectedTrackIds_.size(); i++) {
1371 int index = static_cast<int>(selectedTrackIds_[i]);
1372 if (formatContext_->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
1373 trackIndex = index;
1374 break;
1375 }
1376 }
1377 MEDIA_LOG_D("Seek based on track " PUBLIC_LOG_D32, trackIndex);
1378 auto avStream = formatContext_->streams[trackIndex];
1379 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1380 int64_t ffTime = ConvertTimeToFFmpeg(seekTime * MS_TO_NS, avStream->time_base);
1381 if (!CheckStartTime(formatContext_.get(), avStream, ffTime, seekTime)) {
1382 MEDIA_LOG_E("Get start time from track " PUBLIC_LOG_D32 " failed", trackIndex);
1383 return Status::ERROR_INVALID_OPERATION;
1384 }
1385 realSeekTime = ConvertTimeFromFFmpeg(ffTime, avStream->time_base);
1386 int flag = ConvertFlagsToFFmpeg(avStream, ffTime, mode, seekTime);
1387 MEDIA_LOG_I("Time [" PUBLIC_LOG_U64 "/" PUBLIC_LOG_U64 "/" PUBLIC_LOG_D64 "] flag ["
1388 PUBLIC_LOG_D32 "/" PUBLIC_LOG_D32 "]",
1389 seekTime, ffTime, realSeekTime, static_cast<int32_t>(mode), flag);
1390 auto ret = av_seek_frame(formatContext_.get(), trackIndex, ffTime, flag);
1391 if (formatContext_->pb->error) {
1392 formatContext_->pb->error = 0;
1393 }
1394 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN,
1395 "Call av_seek_frame failed, err: " PUBLIC_LOG_S, AVStrError(ret).c_str());
1396 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1397 cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1398 cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1399 }
1400 return Status::OK;
1401 }
1402
Flush()1403 Status FFmpegDemuxerPlugin::Flush()
1404 {
1405 Status ret = Status::OK;
1406 std::lock_guard<std::shared_mutex> lock(sharedMutex_);
1407 MEDIA_LOG_I("In");
1408 for (size_t i = 0; i < selectedTrackIds_.size(); ++i) {
1409 ret = cacheQueue_.RemoveTrackQueue(selectedTrackIds_[i]);
1410 ret = cacheQueue_.AddTrackQueue(selectedTrackIds_[i]);
1411 }
1412 if (formatContext_) {
1413 avio_flush(formatContext_.get()->pb);
1414 avformat_flush(formatContext_.get());
1415 }
1416 return ret;
1417 }
1418
ResetEosStatus()1419 void FFmpegDemuxerPlugin::ResetEosStatus()
1420 {
1421 MEDIA_LOG_I("In");
1422 formatContext_->pb->eof_reached = 0;
1423 formatContext_->pb->error = 0;
1424 }
1425
ReadSample(uint32_t trackId,std::shared_ptr<AVBuffer> sample)1426 Status FFmpegDemuxerPlugin::ReadSample(uint32_t trackId, std::shared_ptr<AVBuffer> sample)
1427 {
1428 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1429 MediaAVCodec::AVCodecTrace trace("ReadSample");
1430 MEDIA_LOG_D("In");
1431 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1432 FALSE_RETURN_V_MSG_E(!selectedTrackIds_.empty(), Status::ERROR_INVALID_OPERATION, "No track has been selected");
1433 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_INVALID_PARAMETER, "Track has not been selected");
1434 FALSE_RETURN_V_MSG_E(sample != nullptr && sample->memory_!=nullptr, Status::ERROR_INVALID_PARAMETER,
1435 "AVBuffer or memory is nullptr");
1436 Status ret;
1437 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1438 ret = ReadPacketToCacheQueue(trackId);
1439 }
1440 while (!cacheQueue_.HasCache(trackId)) {
1441 ret = ReadPacketToCacheQueue(trackId);
1442 if (ret == Status::END_OF_STREAM) {
1443 MEDIA_LOG_D("Read to end");
1444 }
1445 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1446 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1447 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1448 }
1449 std::lock_guard<std::mutex> lockTrack(*trackMtx_[trackId].get());
1450 auto samplePacket = cacheQueue_.Front(trackId);
1451 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_NULL_POINTER, "Cache packet is nullptr");
1452 if (samplePacket->isEOS) {
1453 ret = SetEosSample(sample);
1454 if (ret == Status::OK) {
1455 MEDIA_LOG_I("Track:" PUBLIC_LOG_D32 " eos [" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "/" PUBLIC_LOG_D64 "]",
1456 trackId, trackDfxInfoMap_[trackId].lastPts,
1457 trackDfxInfoMap_[trackId].lastDurantion, trackDfxInfoMap_[trackId].lastPos);
1458 cacheQueue_.Pop(trackId);
1459 }
1460 return ret;
1461 }
1462 ret = ConvertAVPacketToSample(sample, samplePacket);
1463 if (ret == Status::ERROR_NOT_ENOUGH_DATA) {
1464 return Status::OK;
1465 } else if (ret == Status::OK) {
1466 MEDIA_LOG_D("All partial sample has been copied");
1467 cacheQueue_.Pop(trackId);
1468 }
1469 return ret;
1470 }
1471
GetNextSampleSize(uint32_t trackId,int32_t & size)1472 Status FFmpegDemuxerPlugin::GetNextSampleSize(uint32_t trackId, int32_t& size)
1473 {
1474 std::shared_lock<std::shared_mutex> lock(sharedMutex_);
1475 MediaAVCodec::AVCodecTrace trace("GetNextSampleSize");
1476 MEDIA_LOG_D("In, track " PUBLIC_LOG_D32, trackId);
1477 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_UNKNOWN, "AVFormatContext is nullptr");
1478 FALSE_RETURN_V_MSG_E(TrackIsSelected(trackId), Status::ERROR_UNKNOWN, "Track has not been selected");
1479
1480 Status ret;
1481 if (NeedCombineFrame(trackId) && cacheQueue_.GetCacheSize(trackId) == 1) {
1482 ret = ReadPacketToCacheQueue(trackId);
1483 }
1484 while (!cacheQueue_.HasCache(trackId)) {
1485 ret = ReadPacketToCacheQueue(trackId);
1486 if (ret == Status::END_OF_STREAM) {
1487 MEDIA_LOG_D("Read to end");
1488 }
1489 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_UNKNOWN, ret, "Read from ffmpeg faild");
1490 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_AGAIN, ret, "Read from ffmpeg faild, retry");
1491 FALSE_RETURN_V_MSG_E(ret != Status::ERROR_NO_MEMORY, ret, "Cache size out of limit");
1492 }
1493 std::shared_ptr<SamplePacket> samplePacket = cacheQueue_.Front(trackId);
1494 FALSE_RETURN_V_MSG_E(samplePacket != nullptr, Status::ERROR_UNKNOWN, "Cache sample is nullptr");
1495 if (samplePacket->isEOS) {
1496 MEDIA_LOG_I("Track " PUBLIC_LOG_D32 " eos", trackId);
1497 return Status::END_OF_STREAM;
1498 }
1499 FALSE_RETURN_V_MSG_E(samplePacket->pkts.size() > 0, Status::ERROR_UNKNOWN, "Cache sample is empty");
1500 int totalSize = 0;
1501 for (auto pkt : samplePacket->pkts) {
1502 FALSE_RETURN_V_MSG_E(pkt != nullptr, Status::ERROR_UNKNOWN, "Packet in sample is nullptr");
1503 totalSize += pkt->size;
1504 }
1505 size = totalSize;
1506 return Status::OK;
1507 }
1508
InitPTSandIndexConvert()1509 void FFmpegDemuxerPlugin::InitPTSandIndexConvert()
1510 {
1511 indexToRelativePTSFrameCount_ = 0; // init IndexToRelativePTSFrameCount_
1512 relativePTSToIndexPosition_ = 0; // init RelativePTSToIndexPosition_
1513 indexToRelativePTSMaxHeap_ = std::priority_queue<int64_t>(); // init IndexToRelativePTSMaxHeap_
1514 relativePTSToIndexPTSMin_ = INT64_MAX;
1515 relativePTSToIndexPTSMax_ = INT64_MIN;
1516 relativePTSToIndexRightDiff_ = INT64_MAX;
1517 relativePTSToIndexLeftDiff_ = INT64_MAX;
1518 relativePTSToIndexTempDiff_ = INT64_MAX;
1519 }
1520
GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,const uint64_t relativePresentationTimeUs,uint32_t & index)1521 Status FFmpegDemuxerPlugin::GetIndexByRelativePresentationTimeUs(const uint32_t trackIndex,
1522 const uint64_t relativePresentationTimeUs, uint32_t &index)
1523 {
1524 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1525
1526 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1527 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1528
1529 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1530 bool frameCheck = IsLessMaxReferenceParserFrames(trackIndex);
1531 FALSE_RETURN_V_MSG_E(frameCheck, Status::ERROR_INVALID_DATA, "Frame count exceeds limit");
1532
1533 InitPTSandIndexConvert();
1534
1535 auto avStream = formatContext_->streams[trackIndex];
1536 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1537
1538 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1539 static_cast<int64_t>(relativePresentationTimeUs), index);
1540 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1541
1542 int64_t absolutePTS = static_cast<int64_t>(relativePresentationTimeUs) + absolutePTSIndexZero_;
1543
1544 ret = GetPresentationTimeUsFromFfmpegMOV(RELATIVEPTS_TO_INDEX, trackIndex,
1545 absolutePTS, index);
1546 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1547
1548 if (absolutePTS < relativePTSToIndexPTSMin_ || absolutePTS > relativePTSToIndexPTSMax_) {
1549 MEDIA_LOG_E("Pts is out of range");
1550 return Status::ERROR_INVALID_DATA;
1551 }
1552
1553 if (relativePTSToIndexLeftDiff_ == 0 || relativePTSToIndexRightDiff_ == 0) {
1554 index = relativePTSToIndexPosition_;
1555 } else {
1556 index = relativePTSToIndexLeftDiff_ < relativePTSToIndexRightDiff_ ?
1557 relativePTSToIndexPosition_ - 1 : relativePTSToIndexPosition_;
1558 }
1559 return Status::OK;
1560 }
1561
GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,const uint32_t index,uint64_t & relativePresentationTimeUs)1562 Status FFmpegDemuxerPlugin::GetRelativePresentationTimeUsByIndex(const uint32_t trackIndex,
1563 const uint32_t index, uint64_t &relativePresentationTimeUs)
1564 {
1565 FALSE_RETURN_V_MSG_E(formatContext_ != nullptr, Status::ERROR_NULL_POINTER, "AVFormatContext is nullptr");
1566
1567 FALSE_RETURN_V_MSG_E(FFmpegFormatHelper::GetFileTypeByName(*formatContext_) == FileType::MP4,
1568 Status::ERROR_MISMATCHED_TYPE, "FileType is not MP4");
1569
1570 FALSE_RETURN_V_MSG_E(trackIndex < formatContext_->nb_streams, Status::ERROR_INVALID_DATA, "Track is out of range");
1571 bool frameCheck = IsLessMaxReferenceParserFrames(trackIndex);
1572 FALSE_RETURN_V_MSG_E(frameCheck, Status::ERROR_INVALID_DATA, "Frame count exceeds limit");
1573
1574 InitPTSandIndexConvert();
1575
1576 auto avStream = formatContext_->streams[trackIndex];
1577 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1578
1579 Status ret = GetPresentationTimeUsFromFfmpegMOV(GET_FIRST_PTS, trackIndex,
1580 static_cast<int64_t>(relativePresentationTimeUs), index);
1581 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1582
1583 GetPresentationTimeUsFromFfmpegMOV(INDEX_TO_RELATIVEPTS, trackIndex,
1584 static_cast<int64_t>(relativePresentationTimeUs), index);
1585 FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_UNKNOWN, "Get pts failed");
1586
1587 if (index + 1 > indexToRelativePTSFrameCount_) {
1588 MEDIA_LOG_E("Index is out of range");
1589 return Status::ERROR_INVALID_DATA;
1590 }
1591
1592 int64_t relativepts = indexToRelativePTSMaxHeap_.top() - absolutePTSIndexZero_;
1593 FALSE_RETURN_V_MSG_E(relativepts >= 0, Status::ERROR_INVALID_DATA, "Existence of calculation results less than 0");
1594 relativePresentationTimeUs = static_cast<uint64_t>(relativepts);
1595
1596 return Status::OK;
1597 }
1598
PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1599 Status FFmpegDemuxerPlugin::PTSAndIndexConvertSttsAndCttsProcess(IndexAndPTSConvertMode mode,
1600 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1601 {
1602 uint32_t sttsIndex = 0;
1603 uint32_t cttsIndex = 0;
1604 int64_t pts = 0; // init pts
1605 int64_t dts = 0; // init dts
1606
1607 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1608 int32_t cttsCurNum = 0;
1609
1610 cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1611 while (sttsIndex < avStream->stts_count && cttsIndex < avStream->ctts_count &&
1612 cttsCurNum >= 0 && sttsCurNum >= 0) {
1613 if (cttsCurNum == 0) {
1614 cttsIndex++;
1615 if (cttsIndex >= avStream->ctts_count) {
1616 break;
1617 }
1618 cttsCurNum = static_cast<int32_t>(avStream->ctts_data[cttsIndex].count);
1619 }
1620 cttsCurNum--;
1621 if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1622 ((dts + static_cast<int64_t>(avStream->ctts_data[cttsIndex].duration)) /
1623 static_cast<int64_t>(avStream->time_scale))) {
1624 MEDIA_LOG_E("pts overflow");
1625 return Status::ERROR_INVALID_DATA;
1626 }
1627 double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1628 double ptsTemp = static_cast<double>(dts) + static_cast<double>(avStream->ctts_data[cttsIndex].duration);
1629 pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1630 if (mode == GET_ALL_FRAME_PTS &&
1631 static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1632 MEDIA_LOG_I("PTS list has reached the maximum limit");
1633 break;
1634 }
1635 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1636 sttsCurNum--;
1637 if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1638 MEDIA_LOG_E("dts overflow");
1639 return Status::ERROR_INVALID_DATA;
1640 }
1641 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1642 if (sttsCurNum == 0) {
1643 sttsIndex++;
1644 sttsCurNum = sttsIndex < avStream->stts_count ?
1645 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1646 }
1647 }
1648 return Status::OK;
1649 }
1650
PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,const AVStream * avStream,int64_t absolutePTS,uint32_t index)1651 Status FFmpegDemuxerPlugin::PTSAndIndexConvertOnlySttsProcess(IndexAndPTSConvertMode mode,
1652 const AVStream* avStream, int64_t absolutePTS, uint32_t index)
1653 {
1654 uint32_t sttsIndex = 0;
1655 int64_t pts = 0; // init pts
1656 int64_t dts = 0; // init dts
1657
1658 int32_t sttsCurNum = static_cast<int32_t>(avStream->stts_data[sttsIndex].count);
1659
1660 while (sttsIndex < avStream->stts_count && sttsCurNum >= 0) {
1661 if ((INT64_MAX / 1000 / 1000) < // 1000 is used for converting pts to us
1662 (dts / static_cast<int64_t>(avStream->time_scale))) {
1663 MEDIA_LOG_E("pts overflow");
1664 return Status::ERROR_INVALID_DATA;
1665 }
1666 double timeScaleRate = static_cast<double>(MS_TO_NS) / static_cast<double>(avStream->time_scale);
1667 double ptsTemp = static_cast<double>(dts);
1668 pts = static_cast<int64_t>(ptsTemp * timeScaleRate);
1669 if (mode == GET_ALL_FRAME_PTS &&
1670 static_cast<uint32_t>(ptsListOrg_.size()) >= REFERENCE_PARSER_PTS_LIST_UPPER_LIMIT) {
1671 MEDIA_LOG_I("PTS list has reached the maximum limit");
1672 break;
1673 }
1674 PTSAndIndexConvertSwitchProcess(mode, pts, absolutePTS, index);
1675 sttsCurNum--;
1676 if ((INT64_MAX - dts) < (static_cast<int64_t>(avStream->stts_data[sttsIndex].duration))) {
1677 MEDIA_LOG_E("dts overflow");
1678 return Status::ERROR_INVALID_DATA;
1679 }
1680 dts += static_cast<int64_t>(avStream->stts_data[sttsIndex].duration);
1681 if (sttsCurNum == 0) {
1682 sttsIndex++;
1683 sttsCurNum = sttsIndex < avStream->stts_count ?
1684 static_cast<int32_t>(avStream->stts_data[sttsIndex].count) : 0;
1685 }
1686 }
1687 return Status::OK;
1688 }
1689
GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,uint32_t trackIndex,int64_t absolutePTS,uint32_t index)1690 Status FFmpegDemuxerPlugin::GetPresentationTimeUsFromFfmpegMOV(IndexAndPTSConvertMode mode,
1691 uint32_t trackIndex, int64_t absolutePTS, uint32_t index)
1692 {
1693 auto avStream = formatContext_->streams[trackIndex];
1694 FALSE_RETURN_V_MSG_E(avStream != nullptr, Status::ERROR_NULL_POINTER, "AVStream is nullptr");
1695 FALSE_RETURN_V_MSG_E(avStream->stts_data != nullptr && avStream->stts_count != 0,
1696 Status::ERROR_NULL_POINTER, "AVStream->stts_data is empty");
1697 FALSE_RETURN_V_MSG_E(avStream->time_scale != 0, Status::ERROR_INVALID_DATA, "AVStream->time_scale is zero");
1698 if (mode == GET_ALL_FRAME_PTS) {
1699 ptsListOrg_.clear();
1700 }
1701 return avStream->ctts_data == nullptr ?
1702 PTSAndIndexConvertOnlySttsProcess(mode, avStream, absolutePTS, index) :
1703 PTSAndIndexConvertSttsAndCttsProcess(mode, avStream, absolutePTS, index);
1704 }
1705
PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,int64_t pts,int64_t absolutePTS,uint32_t index)1706 void FFmpegDemuxerPlugin::PTSAndIndexConvertSwitchProcess(IndexAndPTSConvertMode mode,
1707 int64_t pts, int64_t absolutePTS, uint32_t index)
1708 {
1709 switch (mode) {
1710 case GET_FIRST_PTS:
1711 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1712 break;
1713 case INDEX_TO_RELATIVEPTS:
1714 IndexToRelativePTSProcess(pts, index);
1715 break;
1716 case RELATIVEPTS_TO_INDEX:
1717 RelativePTSToIndexProcess(pts, absolutePTS);
1718 break;
1719 case GET_ALL_FRAME_PTS:
1720 absolutePTSIndexZero_ = pts < absolutePTSIndexZero_ ? pts : absolutePTSIndexZero_;
1721 ptsListOrg_.emplace_back(pts);
1722 break;
1723 default:
1724 MEDIA_LOG_E("Wrong mode");
1725 break;
1726 }
1727 }
1728
IndexToRelativePTSProcess(int64_t pts,uint32_t index)1729 void FFmpegDemuxerPlugin::IndexToRelativePTSProcess(int64_t pts, uint32_t index)
1730 {
1731 if (indexToRelativePTSMaxHeap_.size() < index + 1) {
1732 indexToRelativePTSMaxHeap_.push(pts);
1733 } else {
1734 if (pts < indexToRelativePTSMaxHeap_.top()) {
1735 indexToRelativePTSMaxHeap_.pop();
1736 indexToRelativePTSMaxHeap_.push(pts);
1737 }
1738 }
1739 indexToRelativePTSFrameCount_++;
1740 }
1741
RelativePTSToIndexProcess(int64_t pts,int64_t absolutePTS)1742 void FFmpegDemuxerPlugin::RelativePTSToIndexProcess(int64_t pts, int64_t absolutePTS)
1743 {
1744 if (relativePTSToIndexPTSMin_ > pts) {
1745 relativePTSToIndexPTSMin_ = pts;
1746 }
1747 if (relativePTSToIndexPTSMax_ < pts) {
1748 relativePTSToIndexPTSMax_ = pts;
1749 }
1750 relativePTSToIndexTempDiff_ = abs(pts - absolutePTS);
1751 if (pts < absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexLeftDiff_) {
1752 relativePTSToIndexLeftDiff_ = relativePTSToIndexTempDiff_;
1753 }
1754 if (pts >= absolutePTS && relativePTSToIndexTempDiff_ < relativePTSToIndexRightDiff_) {
1755 relativePTSToIndexRightDiff_ = relativePTSToIndexTempDiff_;
1756 }
1757 if (pts < absolutePTS) {
1758 relativePTSToIndexPosition_++;
1759 }
1760 }
1761
CheckCacheDataLimit(uint32_t trackId)1762 Status FFmpegDemuxerPlugin::CheckCacheDataLimit(uint32_t trackId)
1763 {
1764 if (!outOfLimit_) {
1765 auto cacheDataSize = cacheQueue_.GetCacheDataSize(trackId);
1766 if (cacheDataSize > cachelimitSize_) {
1767 MEDIA_LOG_W("Track " PUBLIC_LOG_U32 " cache out of limit: " PUBLIC_LOG_U32 "/" PUBLIC_LOG_U32 ", by user "
1768 PUBLIC_LOG_D32, trackId, cacheDataSize, cachelimitSize_, static_cast<int32_t>(setLimitByUser));
1769 outOfLimit_ = true;
1770 }
1771 }
1772 return Status::OK;
1773 }
1774
SetCacheLimit(uint32_t limitSize)1775 void FFmpegDemuxerPlugin::SetCacheLimit(uint32_t limitSize)
1776 {
1777 setLimitByUser = true;
1778 cachelimitSize_ = limitSize;
1779 }
1780
1781 namespace { // plugin set
1782
IsStartWithID3(const uint8_t * buf,const char * tagName)1783 int IsStartWithID3(const uint8_t *buf, const char *tagName)
1784 {
1785 return buf[POS_0] == tagName[POS_0] &&
1786 buf[POS_1] == tagName[POS_1] &&
1787 buf[POS_2] == tagName[POS_2] &&
1788 buf[POS_3] != POS_FF &&
1789 buf[POS_4] != POS_FF &&
1790 (buf[POS_6] & TAG_MASK) == 0 &&
1791 (buf[POS_7] & TAG_MASK) == 0 &&
1792 (buf[POS_8] & TAG_MASK) == 0 &&
1793 (buf[POS_9] & TAG_MASK) == 0;
1794 }
1795
GetID3TagLen(const uint8_t * buf)1796 int GetID3TagLen(const uint8_t *buf)
1797 {
1798 int32_t len = ((buf[POS_6] & LEN_MASK) << POS_21) + ((buf[POS_7] & LEN_MASK) << POS_14) +
1799 ((buf[POS_8] & LEN_MASK) << POS_7) + (buf[POS_9] & LEN_MASK) +
1800 static_cast<int32_t>(ID3V2_HEADER_SIZE);
1801 if (buf[POS_5] & TAG_VERSION_MASK) {
1802 len += static_cast<int32_t>(ID3V2_HEADER_SIZE);
1803 }
1804 return len;
1805 }
1806
GetConfidence(std::shared_ptr<AVInputFormat> plugin,const std::string & pluginName,std::shared_ptr<DataSource> dataSource,size_t & getData)1807 int32_t GetConfidence(std::shared_ptr<AVInputFormat> plugin, const std::string& pluginName,
1808 std::shared_ptr<DataSource> dataSource, size_t &getData)
1809 {
1810 size_t bufferSize = DEFAULT_SNIFF_SIZE;
1811 uint64_t fileSize = 0;
1812 Status getFileSize = dataSource->GetSize(fileSize);
1813 if (getFileSize == Status::OK) {
1814 bufferSize = (bufferSize < fileSize) ? bufferSize : fileSize;
1815 }
1816 std::vector<uint8_t> buff(bufferSize + AVPROBE_PADDING_SIZE); // fix ffmpeg probe crash, refer to tools/probetest.c
1817 auto bufferInfo = std::make_shared<Buffer>();
1818 auto bufData = bufferInfo->WrapMemory(buff.data(), bufferSize, bufferSize);
1819 FALSE_RETURN_V_MSG_E(bufferInfo->GetMemory() != nullptr, -1,
1820 "Alloc buffer failed for " PUBLIC_LOG_S, pluginName.c_str());
1821 Status ret = Status::OK;
1822 {
1823 std::string traceName = "Sniff_" + pluginName + "_Readat";
1824 MediaAVCodec::AVCodecTrace trace(traceName.c_str());
1825 ret = dataSource->ReadAt(0, bufferInfo, bufferSize);
1826 }
1827 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1828 getData = bufferInfo->GetMemory()->GetSize();
1829 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1830 if (getFileSize == Status::OK && getData > ID3V2_HEADER_SIZE && IsStartWithID3(buff.data(), "ID3")) {
1831 int32_t id3Len = GetID3TagLen(buff.data());
1832 // id3 tag length is out of file, or file just contains id3 tag, no valid data.
1833 FALSE_RETURN_V_MSG_E(id3Len >= 0 && static_cast<uint64_t>(id3Len) < fileSize, -1,
1834 "File data error for " PUBLIC_LOG_S, pluginName.c_str());
1835 if (id3Len > 0) {
1836 uint64_t remainSize = fileSize - static_cast<uint64_t>(id3Len);
1837 bufferSize = (bufferSize < remainSize) ? bufferSize : remainSize;
1838 int resetRet = memset_s(buff.data(), bufferSize, 0, bufferSize);
1839 FALSE_RETURN_V_MSG_E(resetRet == EOK, -1, "Reset buff failed for " PUBLIC_LOG_S, pluginName.c_str());
1840 ret = dataSource->ReadAt(id3Len, bufferInfo, bufferSize);
1841 FALSE_RETURN_V_MSG_E(ret == Status::OK, -1, "Read probe data failed for " PUBLIC_LOG_S, pluginName.c_str());
1842 getData = bufferInfo->GetMemory()->GetSize();
1843 FALSE_RETURN_V_MSG_E(getData > 0, -1, "No data for sniff " PUBLIC_LOG_S, pluginName.c_str());
1844 }
1845 }
1846 AVProbeData probeData{"", buff.data(), static_cast<int32_t>(getData), ""};
1847 return plugin->read_probe(&probeData);
1848 }
1849
Sniff(const std::string & pluginName,std::shared_ptr<DataSource> dataSource)1850 int Sniff(const std::string& pluginName, std::shared_ptr<DataSource> dataSource)
1851 {
1852 FALSE_RETURN_V_MSG_E(!pluginName.empty(), 0, "Plugin name is empty");
1853 FALSE_RETURN_V_MSG_E(dataSource != nullptr, 0, "DataSource is nullptr");
1854 std::shared_ptr<AVInputFormat> plugin;
1855 {
1856 std::lock_guard<std::mutex> lock(g_mtx);
1857 plugin = g_pluginInputFormat[pluginName];
1858 }
1859 FALSE_RETURN_V_MSG_E((plugin != nullptr && plugin->read_probe), 0,
1860 "Get plugin for " PUBLIC_LOG_S " failed", pluginName.c_str());
1861 size_t getData = 0;
1862 int confidence = GetConfidence(plugin, pluginName, dataSource, getData);
1863 if (confidence < 0) {
1864 return 0;
1865 }
1866 if (StartWith(plugin->name, "mp3") && confidence > 0 && confidence <= MP3_PROBE_SCORE_LIMIT) {
1867 MEDIA_LOG_W("Score " PUBLIC_LOG_D32 " is too low", confidence);
1868 confidence = 0;
1869 }
1870 if (getData < DEFAULT_SNIFF_SIZE || confidence > 0) {
1871 MEDIA_LOG_I("Sniff:" PUBLIC_LOG_S "[" PUBLIC_LOG_ZU "/" PUBLIC_LOG_D32 "]", plugin->name, getData, confidence);
1872 }
1873 return confidence;
1874 }
1875
ReplaceDelimiter(const std::string & delmiters,char newDelimiter,std::string & str)1876 void ReplaceDelimiter(const std::string& delmiters, char newDelimiter, std::string& str)
1877 {
1878 MEDIA_LOG_D("Reset from [" PUBLIC_LOG_S "]", str.c_str());
1879 for (auto it = str.begin(); it != str.end(); ++it) {
1880 if (delmiters.find(newDelimiter) != std::string::npos) {
1881 *it = newDelimiter;
1882 }
1883 }
1884 MEDIA_LOG_D("Reset to [" PUBLIC_LOG_S "]", str.c_str());
1885 };
1886
RegisterPlugins(const std::shared_ptr<Register> & reg)1887 Status RegisterPlugins(const std::shared_ptr<Register>& reg)
1888 {
1889 MEDIA_LOG_I("In");
1890 FALSE_RETURN_V_MSG_E(reg != nullptr, Status::ERROR_INVALID_PARAMETER, "Register is nullptr");
1891 std::lock_guard<std::mutex> lock(g_mtx);
1892 const AVInputFormat* plugin = nullptr;
1893 void* i = nullptr;
1894 while ((plugin = av_demuxer_iterate(&i))) {
1895 if (plugin == nullptr) {
1896 continue;
1897 }
1898 MEDIA_LOG_D("Check ffmpeg demuxer " PUBLIC_LOG_S "[" PUBLIC_LOG_S "]", plugin->name, plugin->long_name);
1899 if (plugin->long_name != nullptr &&
1900 !strncmp(plugin->long_name, "pcm ", STR_MAX_LEN)) {
1901 continue;
1902 }
1903 if (!IsInputFormatSupported(plugin->name)) {
1904 continue;
1905 }
1906
1907 std::string pluginName = "avdemux_" + std::string(plugin->name);
1908 ReplaceDelimiter(".,|-<> ", '_', pluginName);
1909
1910 DemuxerPluginDef regInfo;
1911 regInfo.name = pluginName;
1912 regInfo.description = "ffmpeg demuxer plugin";
1913 regInfo.rank = RANK_MAX;
1914 regInfo.AddExtensions(SplitString(plugin->extensions, ','));
1915 g_pluginInputFormat[pluginName] =
1916 std::shared_ptr<AVInputFormat>(const_cast<AVInputFormat*>(plugin), [](void*) {});
1917 auto func = [](const std::string& name) -> std::shared_ptr<DemuxerPlugin> {
1918 return std::make_shared<FFmpegDemuxerPlugin>(name);
1919 };
1920 regInfo.SetCreator(func);
1921 regInfo.SetSniffer(Sniff);
1922 auto ret = reg->AddPlugin(regInfo);
1923 if (ret != Status::OK) {
1924 MEDIA_LOG_E("Add plugin failed, err=" PUBLIC_LOG_D32, static_cast<int>(ret));
1925 } else {
1926 MEDIA_LOG_D("Add plugin " PUBLIC_LOG_S, pluginName.c_str());
1927 }
1928 }
1929 FALSE_RETURN_V_MSG_E(!g_pluginInputFormat.empty(), Status::ERROR_UNKNOWN, "Can not load any ffmpeg demuxer");
1930 return Status::OK;
1931 }
1932 } // namespace
__anon204aac900a02null1933 PLUGIN_DEFINITION(FFmpegDemuxer, LicenseType::LGPL, RegisterPlugins, [] {});
1934 } // namespace Ffmpeg
1935 } // namespace Plugins
1936 } // namespace Media
1937 } // namespace OHOS
1938