1 /*
2 * Copyright (C) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "audio_ffmpeg_decoder_plugin.h"
17 #include "avcodec_dfx.h"
18 #include "avcodec_errors.h"
19 #include "avcodec_log.h"
20 #include "media_description.h"
21 #include "ffmpeg_converter.h"
22 #include "securec.h"
23
24 namespace {
25 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AvCodec-AudioFfmpegDecoderPlugin"};
26 constexpr uint8_t LOGD_FREQUENCY = 5;
27 } // namespace
28
29 namespace OHOS {
30 namespace MediaAVCodec {
AudioFfmpegDecoderPlugin()31 AudioFfmpegDecoderPlugin::AudioFfmpegDecoderPlugin()
32 : hasExtra_(false),
33 maxInputSize_(-1),
34 bufferNum_(1),
35 bufferIndex_(1),
36 preBufferGroupPts_(0),
37 curBufferGroupPts_(0),
38 bufferGroupPtsDistance(0),
39 avCodec_(nullptr),
40 avCodecContext_(nullptr),
41 cachedFrame_(nullptr),
42 avPacket_(nullptr),
43 resample_(nullptr),
44 needResample_(false),
45 destFmt_(AV_SAMPLE_FMT_NONE)
46 {
47 }
48
~AudioFfmpegDecoderPlugin()49 AudioFfmpegDecoderPlugin::~AudioFfmpegDecoderPlugin()
50 {
51 AVCODEC_LOGI("AudioFfmpegDecoderPlugin deconstructor running.");
52 CloseCtxLocked();
53 if (avCodecContext_ != nullptr) {
54 avCodecContext_.reset();
55 avCodecContext_ = nullptr;
56 }
57 }
58
ProcessSendData(const std::shared_ptr<AudioBufferInfo> & inputBuffer)59 int32_t AudioFfmpegDecoderPlugin::ProcessSendData(const std::shared_ptr<AudioBufferInfo> &inputBuffer)
60 {
61 std::lock_guard<std::mutex> lock(avMutext_);
62 if (avCodecContext_ == nullptr) {
63 AVCODEC_LOGE("avCodecContext_ is nullptr");
64 return AVCodecServiceErrCode::AVCS_ERR_INVALID_OPERATION;
65 }
66 return SendBuffer(inputBuffer);
67 }
68
AVStrError(int errnum)69 static std::string AVStrError(int errnum)
70 {
71 char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
72 av_strerror(errnum, errbuf, AV_ERROR_MAX_STRING_SIZE);
73 return std::string(errbuf);
74 }
75
GetMaxInputSize() const76 int32_t AudioFfmpegDecoderPlugin::GetMaxInputSize() const noexcept
77 {
78 return maxInputSize_;
79 }
80
HasExtraData() const81 bool AudioFfmpegDecoderPlugin::HasExtraData() const noexcept
82 {
83 return hasExtra_;
84 }
85
SendBuffer(const std::shared_ptr<AudioBufferInfo> & inputBuffer)86 int32_t AudioFfmpegDecoderPlugin::SendBuffer(const std::shared_ptr<AudioBufferInfo> &inputBuffer)
87 {
88 if (!inputBuffer) {
89 AVCODEC_LOGE("inputBuffer is nullptr");
90 return AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL;
91 }
92 auto attr = inputBuffer->GetBufferAttr();
93 if (!inputBuffer->CheckIsEos()) {
94 auto memory = inputBuffer->GetBuffer();
95 const uint8_t *ptr = memory->GetBase();
96 if (attr.size <= 0) {
97 AVCODEC_LOGE("send input buffer is less than 0. size:%{public}d", attr.size);
98 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
99 }
100 if (attr.size > memory->GetSize()) {
101 AVCODEC_LOGE("send input buffer is > allocate size. size:%{public}d,allocate size:%{public}d", attr.size,
102 memory->GetSize());
103 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
104 }
105 avPacket_->size = attr.size;
106 avPacket_->data = const_cast<uint8_t *>(ptr);
107 avPacket_->pts = attr.presentationTimeUs;
108 } else {
109 avPacket_->size = 0;
110 avPacket_->data = nullptr;
111 avPacket_->pts = attr.presentationTimeUs;
112 }
113 AVCODEC_LOGD_LIMIT(LOGD_FREQUENCY, "SendBuffer buffer size:%{public}u,name:%{public}s", attr.size, name_.data());
114 auto ret = avcodec_send_packet(avCodecContext_.get(), avPacket_.get());
115 av_packet_unref(avPacket_.get());
116 if (ret == 0) {
117 return AVCodecServiceErrCode::AVCS_ERR_OK;
118 } else if (ret == AVERROR(EAGAIN)) {
119 AVCODEC_LOGW("skip this frame because data not enough, msg:%{public}s", AVStrError(ret).data());
120 return AVCodecServiceErrCode::AVCS_ERR_NOT_ENOUGH_DATA;
121 } else if (ret == AVERROR_EOF) {
122 AVCODEC_LOGW("eos send frame, msg:%{public}s", AVStrError(ret).data());
123 return AVCodecServiceErrCode::AVCS_ERR_END_OF_STREAM;
124 } else if (ret == AVERROR_INVALIDDATA) {
125 AVCODEC_LOGE("ffmpeg error message, msg:%{public}s", AVStrError(ret).data());
126 return AVCodecServiceErrCode::AVCS_ERR_INVALID_DATA;
127 } else {
128 AVCODEC_LOGE("ffmpeg error message, msg:%{public}s", AVStrError(ret).data());
129 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
130 }
131 }
132
ProcessRecieveData(std::shared_ptr<AudioBufferInfo> & outBuffer)133 int32_t AudioFfmpegDecoderPlugin::ProcessRecieveData(std::shared_ptr<AudioBufferInfo> &outBuffer)
134 {
135 std::lock_guard<std::mutex> l(avMutext_);
136 if (!outBuffer) {
137 AVCODEC_LOGE("outBuffer is nullptr");
138 return AVCodecServiceErrCode::AVCS_ERR_INVALID_VAL;
139 }
140 if (avCodecContext_ == nullptr) {
141 AVCODEC_LOGE("avCodecContext_ is nullptr");
142 return AVCodecServiceErrCode::AVCS_ERR_INVALID_OPERATION;
143 }
144 return ReceiveBuffer(outBuffer);
145 }
146
ReceiveBuffer(std::shared_ptr<AudioBufferInfo> & outBuffer)147 int32_t AudioFfmpegDecoderPlugin::ReceiveBuffer(std::shared_ptr<AudioBufferInfo> &outBuffer)
148 {
149 auto ret = avcodec_receive_frame(avCodecContext_.get(), cachedFrame_.get());
150 int32_t status;
151 if (ret >= 0) {
152 AVCODEC_LOGD_LIMIT(LOGD_FREQUENCY, "receive one frame");
153 if (cachedFrame_->pts != AV_NOPTS_VALUE) {
154 preBufferGroupPts_ = curBufferGroupPts_;
155 curBufferGroupPts_ = cachedFrame_->pts;
156 if (bufferGroupPtsDistance == 0) {
157 bufferGroupPtsDistance = abs(curBufferGroupPts_ - preBufferGroupPts_);
158 }
159 if (bufferIndex_ >= bufferNum_) {
160 bufferNum_ = bufferIndex_;
161 }
162 bufferIndex_ = 1;
163 } else {
164 bufferIndex_++;
165 if (abs(curBufferGroupPts_ - preBufferGroupPts_) > bufferGroupPtsDistance) {
166 cachedFrame_->pts = curBufferGroupPts_;
167 preBufferGroupPts_ = curBufferGroupPts_;
168 } else {
169 cachedFrame_->pts =
170 curBufferGroupPts_ + abs(curBufferGroupPts_ - preBufferGroupPts_) * (bufferIndex_ - 1) / bufferNum_;
171 }
172 }
173 status = ReceiveFrameSucc(outBuffer);
174 } else if (ret == AVERROR_EOF) {
175 AVCODEC_LOGI("eos received");
176 outBuffer->SetEos(true);
177 avcodec_flush_buffers(avCodecContext_.get());
178 status = AVCodecServiceErrCode::AVCS_ERR_END_OF_STREAM;
179 } else if (ret == AVERROR(EAGAIN)) {
180 AVCODEC_LOGW("audio decoder not enough data");
181 status = AVCodecServiceErrCode::AVCS_ERR_NOT_ENOUGH_DATA;
182 } else {
183 AVCODEC_LOGE("audio decoder receive unknow error,ffmpeg error message:%{public}s", AVStrError(ret).data());
184 status = AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
185 }
186 av_frame_unref(cachedFrame_.get());
187 AVCODEC_LOGD_LIMIT(LOGD_FREQUENCY, "end received");
188 return status;
189 }
190
ConvertPlanarFrame(std::shared_ptr<AudioBufferInfo> & outBuffer)191 int32_t AudioFfmpegDecoderPlugin::ConvertPlanarFrame(std::shared_ptr<AudioBufferInfo> &outBuffer)
192 {
193 convertedFrame_ = std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame *fp) { av_frame_free(&fp); });
194 if (convertedFrame_ == nullptr) {
195 AVCODEC_LOGE("av_frame_alloc failed");
196 return AVCodecServiceErrCode::AVCS_ERR_NO_MEMORY;
197 }
198 if (resample_->ConvertFrame(convertedFrame_.get(), cachedFrame_.get()) != AVCodecServiceErrCode::AVCS_ERR_OK) {
199 AVCODEC_LOGE("convert frame failed");
200 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
201 }
202 return AVCodecServiceErrCode::AVCS_ERR_OK;
203 }
204
ReceiveFrameSucc(std::shared_ptr<AudioBufferInfo> & outBuffer)205 int32_t AudioFfmpegDecoderPlugin::ReceiveFrameSucc(std::shared_ptr<AudioBufferInfo> &outBuffer)
206 {
207 auto outFrame = cachedFrame_;
208 if (needResample_ && avCodecContext_->channels > 1) {
209 if (ConvertPlanarFrame(outBuffer) != AVCodecServiceErrCode::AVCS_ERR_OK) {
210 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
211 }
212 outFrame = convertedFrame_;
213 }
214 auto ioInfoMem = outBuffer->GetBuffer();
215 int32_t bytePerSample = av_get_bytes_per_sample(static_cast<AVSampleFormat>(outFrame->format));
216 int32_t outputSize = outFrame->nb_samples * bytePerSample * outFrame->channels;
217 AVCODEC_LOGD_LIMIT(LOGD_FREQUENCY, "ReceiveFrameSucc buffer real size:%{public}u,size:%{public}u, name:%{public}s",
218 outputSize, ioInfoMem->GetSize(), name_.data());
219 if (ioInfoMem->GetSize() < outputSize) {
220 AVCODEC_LOGE("output buffer size is not enough,output size:%{public}d", outputSize);
221 return AVCodecServiceErrCode::AVCS_ERR_NO_MEMORY;
222 }
223
224 ioInfoMem->Write(outFrame->data[0], outputSize);
225
226 if (outBuffer->CheckIsFirstFrame()) {
227 format_.PutIntValue(MediaDescriptionKey::MD_KEY_BITS_PER_CODED_SAMPLE,
228 FFMpegConverter::ConvertFFMpegToOHAudioFormat(avCodecContext_->sample_fmt));
229 auto layout = FFMpegConverter::ConvertFFToOHAudioChannelLayout(avCodecContext_->channel_layout);
230 AVCODEC_LOGI("recode output description,layout:%{public}s",
231 FFMpegConverter::ConvertOHAudioChannelLayoutToString(layout).data());
232 format_.PutLongValue(MediaDescriptionKey::MD_KEY_CHANNEL_LAYOUT, static_cast<uint64_t>(layout));
233 }
234 auto attr = outBuffer->GetBufferAttr();
235 attr.presentationTimeUs = static_cast<uint64_t>(cachedFrame_->pts);
236 attr.size = outputSize;
237 outBuffer->SetBufferAttr(attr);
238 return AVCodecServiceErrCode::AVCS_ERR_OK;
239 }
240
Reset()241 int32_t AudioFfmpegDecoderPlugin::Reset()
242 {
243 std::lock_guard<std::mutex> lock(avMutext_);
244 CloseCtxLocked();
245 if (avCodecContext_ != nullptr) {
246 avCodecContext_.reset();
247 avCodecContext_ = nullptr;
248 }
249 return AVCodecServiceErrCode::AVCS_ERR_OK;
250 }
251
Release()252 int32_t AudioFfmpegDecoderPlugin::Release()
253 {
254 std::lock_guard<std::mutex> lock(avMutext_);
255 auto ret = CloseCtxLocked();
256 if (avCodecContext_ != nullptr) {
257 avCodecContext_.reset();
258 avCodecContext_ = nullptr;
259 }
260 return ret;
261 }
262
Flush()263 int32_t AudioFfmpegDecoderPlugin::Flush()
264 {
265 std::lock_guard<std::mutex> lock(avMutext_);
266 if (avCodecContext_ != nullptr) {
267 avcodec_flush_buffers(avCodecContext_.get());
268 }
269 return AVCodecServiceErrCode::AVCS_ERR_OK;
270 }
271
AllocateContext(const std::string & name)272 int32_t AudioFfmpegDecoderPlugin::AllocateContext(const std::string &name)
273 {
274 {
275 std::lock_guard<std::mutex> lock(avMutext_);
276 avCodec_ = std::shared_ptr<AVCodec>(const_cast<AVCodec *>(avcodec_find_decoder_by_name(name.c_str())),
277 [](AVCodec *ptr) { (void)ptr; });
278 cachedFrame_ = std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame *fp) { av_frame_free(&fp); });
279 }
280 if (avCodec_ == nullptr) {
281 AVCODEC_LOGE("AllocateContext fail,parameter avcodec is nullptr.");
282 return AVCodecServiceErrCode::AVCS_ERR_INVALID_OPERATION;
283 }
284 name_ = name;
285 AVCodecContext *context = nullptr;
286 {
287 std::lock_guard<std::mutex> lock(avMutext_);
288 context = avcodec_alloc_context3(avCodec_.get());
289
290 avCodecContext_ = std::shared_ptr<AVCodecContext>(context, [](AVCodecContext *ptr) {
291 ptr->extradata = nullptr;
292 ptr->extradata_size = 0;
293 avcodec_free_context(&ptr);
294 avcodec_close(ptr);
295 });
296 av_log_set_level(AV_LOG_ERROR);
297 }
298 return AVCodecServiceErrCode::AVCS_ERR_OK;
299 }
300
InitContext(const Format & format)301 int32_t AudioFfmpegDecoderPlugin::InitContext(const Format &format)
302 {
303 format_ = format;
304 format_.GetIntValue(MediaDescriptionKey::MD_KEY_CHANNEL_COUNT, avCodecContext_->channels);
305 if (avCodecContext_->channels <= 0) {
306 return AVCodecServiceErrCode::AVCS_ERR_CONFIGURE_MISMATCH_CHANNEL_COUNT;
307 }
308 format_.GetIntValue(MediaDescriptionKey::MD_KEY_SAMPLE_RATE, avCodecContext_->sample_rate);
309 if (avCodecContext_->sample_rate <= 0) {
310 return AVCodecServiceErrCode::AVCS_ERR_MISMATCH_SAMPLE_RATE;
311 }
312 format_.GetLongValue(MediaDescriptionKey::MD_KEY_BITRATE, avCodecContext_->bit_rate);
313 format_.GetIntValue(MediaDescriptionKey::MD_KEY_MAX_INPUT_SIZE, maxInputSize_);
314
315 size_t extraSize;
316 if (format_.GetBuffer(MediaDescriptionKey::MD_KEY_CODEC_CONFIG, &avCodecContext_->extradata, extraSize)) {
317 avCodecContext_->extradata_size = extraSize;
318 hasExtra_ = true;
319 }
320
321 avCodecContext_->sample_fmt = AV_SAMPLE_FMT_S16;
322 avCodecContext_->request_sample_fmt = avCodecContext_->sample_fmt;
323 avCodecContext_->workaround_bugs =
324 static_cast<uint32_t>(avCodecContext_->workaround_bugs) | static_cast<uint32_t>(FF_BUG_AUTODETECT);
325 avCodecContext_->err_recognition = 1;
326 return AVCodecServiceErrCode::AVCS_ERR_OK;
327 }
328
OpenContext()329 int32_t AudioFfmpegDecoderPlugin::OpenContext()
330 {
331 avPacket_ = std::shared_ptr<AVPacket>(av_packet_alloc(), [](AVPacket *ptr) { av_packet_free(&ptr); });
332 {
333 std::lock_guard<std::mutex> lock(avMutext_);
334 auto res = avcodec_open2(avCodecContext_.get(), avCodec_.get(), nullptr);
335 if (res != 0) {
336 AVCODEC_LOGE("avcodec open error %{public}s", AVStrError(res).c_str());
337 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
338 }
339
340 if (InitResample() != AVCodecServiceErrCode::AVCS_ERR_OK) {
341 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
342 }
343 }
344 return AVCodecServiceErrCode::AVCS_ERR_OK;
345 }
346
InitResample()347 int32_t AudioFfmpegDecoderPlugin::InitResample()
348 {
349 if (needResample_) {
350 ResamplePara resamplePara = {
351 .channels = avCodecContext_->channels,
352 .sampleRate = avCodecContext_->sample_rate,
353 .bitsPerSample = 0,
354 .channelLayout = avCodecContext_->channel_layout,
355 .srcFmt = avCodecContext_->sample_fmt,
356 .destSamplesPerFrame = 0,
357 .destFmt = destFmt_,
358 };
359 convertedFrame_ = std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame *fp) { av_frame_free(&fp); });
360 resample_ = std::make_shared<AudioResample>();
361 if (resample_->InitSwrContext(resamplePara) != AVCodecServiceErrCode::AVCS_ERR_OK) {
362 AVCODEC_LOGE("Resample init failed.");
363 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
364 }
365 }
366 return AVCodecServiceErrCode::AVCS_ERR_OK;
367 }
368
GetFormat() const369 Format AudioFfmpegDecoderPlugin::GetFormat() const noexcept
370 {
371 return format_;
372 }
373
GetCodecContext() const374 std::shared_ptr<AVCodecContext> AudioFfmpegDecoderPlugin::GetCodecContext() const noexcept
375 {
376 return avCodecContext_;
377 }
378
GetCodecAVPacket() const379 std::shared_ptr<AVPacket> AudioFfmpegDecoderPlugin::GetCodecAVPacket() const noexcept
380 {
381 return avPacket_;
382 }
383
GetCodecCacheFrame() const384 std::shared_ptr<AVFrame> AudioFfmpegDecoderPlugin::GetCodecCacheFrame() const noexcept
385 {
386 return cachedFrame_;
387 }
388
CloseCtxLocked()389 int32_t AudioFfmpegDecoderPlugin::CloseCtxLocked()
390 {
391 if (avCodecContext_ != nullptr) {
392 auto res = avcodec_close(avCodecContext_.get());
393 if (res != 0) {
394 AVCODEC_LOGE("avcodec close failed, res=%{public}d", res);
395 return AVCodecServiceErrCode::AVCS_ERR_UNKNOWN;
396 }
397 }
398 return AVCodecServiceErrCode::AVCS_ERR_OK;
399 }
400
EnableResample(AVSampleFormat destFmt)401 void AudioFfmpegDecoderPlugin::EnableResample(AVSampleFormat destFmt)
402 {
403 needResample_ = true;
404 destFmt_ = destFmt;
405 }
406 } // namespace MediaAVCodec
407 } // namespace OHOS