1 /*
2 * Copyright (c) 2021-2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifdef VIDEO_SUPPORT
17
18 #define HST_LOG_TAG "FfmpegVideoDecoderPlugin"
19
20 #include "video_ffmpeg_decoder_plugin.h"
21 #include <cstring>
22 #include <map>
23 #include <set>
24 #include "plugin/common/plugin_caps_builder.h"
25 #include "plugins/ffmpeg_adapter/utils/ffmpeg_utils.h"
26 #include "plugin/common/surface_memory.h"
27
28 namespace {
29 // register plugins
30 using namespace OHOS::Media::Plugin;
31 using namespace Ffmpeg;
32 void UpdatePluginDefinition(const AVCodec* codec, CodecPluginDef& definition);
33
34 std::map<std::string, std::shared_ptr<const AVCodec>> codecMap;
35
36 constexpr size_t BUFFER_QUEUE_SIZE = 8;
37 constexpr int32_t STRIDE_ALIGN = 16;
38
39 std::set<AVCodecID> supportedCodec = {AV_CODEC_ID_H264};
40
VideoFfmpegDecoderCreator(const std::string & name)41 std::shared_ptr<CodecPlugin> VideoFfmpegDecoderCreator(const std::string& name)
42 {
43 return std::make_shared<VideoFfmpegDecoderPlugin>(name);
44 }
45
RegisterVideoDecoderPlugins(const std::shared_ptr<Register> & reg)46 Status RegisterVideoDecoderPlugins(const std::shared_ptr<Register>& reg)
47 {
48 const AVCodec* codec = nullptr;
49 void* iter = nullptr;
50 MEDIA_LOG_I("registering video decoders");
51 while ((codec = av_codec_iterate(&iter))) {
52 if (!av_codec_is_decoder(codec) || codec->type != AVMEDIA_TYPE_VIDEO) {
53 continue;
54 }
55 if (supportedCodec.find(codec->id) == supportedCodec.end()) {
56 MEDIA_LOG_DD("codec " PUBLIC_LOG_S "(" PUBLIC_LOG_S ") is not supported right now",
57 codec->name, codec->long_name);
58 continue;
59 }
60 CodecPluginDef definition;
61 definition.name = "videodecoder_" + std::string(codec->name);
62 definition.codecType = CodecType::VIDEO_DECODER;
63 definition.rank = 100; // 100
64 definition.creator = VideoFfmpegDecoderCreator;
65 UpdatePluginDefinition(codec, definition);
66 // do not delete the codec in the deleter
67 codecMap[definition.name] = std::shared_ptr<AVCodec>(const_cast<AVCodec*>(codec), [](void* ptr) {});
68 if (reg->AddPlugin(definition) != Status::OK) {
69 MEDIA_LOG_W("register plugin " PUBLIC_LOG_S "(" PUBLIC_LOG_S ") failed",
70 codec->name, codec->long_name);
71 }
72 }
73 return Status::OK;
74 }
75
UnRegisterVideoDecoderPlugins()76 void UnRegisterVideoDecoderPlugins()
77 {
78 codecMap.clear();
79 }
80
UpdateInCaps(const AVCodec * codec,CodecPluginDef & definition)81 void UpdateInCaps(const AVCodec* codec, CodecPluginDef& definition)
82 {
83 CapabilityBuilder incapBuilder;
84 switch (codec->id) {
85 case AV_CODEC_ID_H264:
86 incapBuilder.SetMime(OHOS::Media::MEDIA_MIME_VIDEO_H264);
87 incapBuilder.SetVideoBitStreamFormatList({VideoBitStreamFormat::AVC1, VideoBitStreamFormat::ANNEXB});
88 break;
89 default:
90 incapBuilder.SetMime("video/unknown");
91 MEDIA_LOG_I("codec is not supported right now");
92 break;
93 }
94 definition.inCaps.push_back(incapBuilder.Build());
95 }
96
UpdateOutCaps(const AVCodec * codec,CodecPluginDef & definition)97 void UpdateOutCaps(const AVCodec* codec, CodecPluginDef& definition)
98 {
99 CapabilityBuilder outcapBuilder;
100 outcapBuilder.SetMime(OHOS::Media::MEDIA_MIME_VIDEO_RAW);
101 if (codec->pix_fmts != nullptr) {
102 DiscreteCapability<VideoPixelFormat> values;
103 size_t index = 0;
104 for (index = 0; codec->pix_fmts[index] != 0; ++index) {
105 auto supportFormat = ConvertPixelFormatFromFFmpeg(codec->pix_fmts[index]);
106 if (supportFormat != VideoPixelFormat::UNKNOWN) {
107 values.push_back(supportFormat);
108 }
109 }
110 if (index) {
111 outcapBuilder.SetVideoPixelFormatList(values);
112 }
113 }
114 definition.outCaps.push_back(outcapBuilder.Build());
115 }
116
UpdatePluginDefinition(const AVCodec * codec,CodecPluginDef & definition)117 void UpdatePluginDefinition(const AVCodec* codec, CodecPluginDef& definition)
118 {
119 UpdateInCaps(codec, definition);
120 UpdateOutCaps(codec, definition);
121 }
122 } // namespace
123
124 PLUGIN_DEFINITION(FFmpegVideoDecoders, LicenseType::LGPL, RegisterVideoDecoderPlugins, UnRegisterVideoDecoderPlugins);
125
126 namespace OHOS {
127 namespace Media {
128 namespace Plugin {
VideoFfmpegDecoderPlugin(std::string name)129 VideoFfmpegDecoderPlugin::VideoFfmpegDecoderPlugin(std::string name)
130 : CodecPlugin(std::move(name)), outBufferQ_("vdecPluginQueue", BUFFER_QUEUE_SIZE)
131 {
132 for (int32_t i = 0; i < AV_NUM_DATA_POINTERS; i++) {
133 scaleData_[i] = nullptr;
134 scaleLineSize_[i] = 0;
135 }
136 isAllocScaleData_ = false;
137 }
138
Init()139 Status VideoFfmpegDecoderPlugin::Init()
140 {
141 OSAL::ScopedLock l(avMutex_);
142 auto iter = codecMap.find(pluginName_);
143 if (iter == codecMap.end()) {
144 MEDIA_LOG_W("cannot find codec with name " PUBLIC_LOG_S, pluginName_.c_str());
145 return Status::ERROR_UNSUPPORTED_FORMAT;
146 }
147 avCodec_ = iter->second;
148 cachedFrame_ = std::shared_ptr<AVFrame>(av_frame_alloc(), [](AVFrame* fp) { av_frame_free(&fp); });
149 videoDecParams_[Tag::REQUIRED_OUT_BUFFER_CNT] = (uint32_t)BUFFER_QUEUE_SIZE;
150 if (!decodeTask_) {
151 decodeTask_ = std::make_shared<OHOS::Media::OSAL::Task>("videoFfmpegDecThread");
152 decodeTask_->RegisterHandler([this] { ReceiveFrameBuffer(); });
153 }
154 state_ = State::INITIALIZED;
155 MEDIA_LOG_I("Init success");
156 return Status::OK;
157 }
158
Deinit()159 Status VideoFfmpegDecoderPlugin::Deinit()
160 {
161 OSAL::ScopedLock l(avMutex_);
162 avCodec_.reset();
163 cachedFrame_.reset();
164 ResetLocked();
165 if (decodeTask_) {
166 decodeTask_->Stop();
167 decodeTask_.reset();
168 }
169 state_ = State::DESTROYED;
170 return Status::OK;
171 }
172
SetParameter(Tag tag,const ValueType & value)173 Status VideoFfmpegDecoderPlugin::SetParameter(Tag tag, const ValueType& value)
174 {
175 OSAL::ScopedLock l(avMutex_);
176 videoDecParams_.insert(std::make_pair(tag, value));
177 return Status::OK;
178 }
179
GetParameter(Tag tag,ValueType & value)180 Status VideoFfmpegDecoderPlugin::GetParameter(Tag tag, ValueType& value)
181 {
182 OSAL::ScopedLock l(avMutex_);
183 auto res = videoDecParams_.find(tag);
184 if (res != videoDecParams_.end()) {
185 value = res->second;
186 return Status::OK;
187 }
188 return Status::ERROR_INVALID_PARAMETER;
189 }
190
191 template <typename T>
FindInParameterMapThenAssignLocked(Tag tag,T & assign)192 void VideoFfmpegDecoderPlugin::FindInParameterMapThenAssignLocked(Tag tag, T& assign)
193 {
194 auto iter = videoDecParams_.find(tag);
195 if (iter != videoDecParams_.end() && iter->second.SameTypeWith(typeid(T))) {
196 assign = Plugin::AnyCast<T>(iter->second);
197 } else {
198 MEDIA_LOG_W("parameter " PUBLIC_LOG_D32 " is not found or type mismatch", static_cast<int32_t>(tag));
199 }
200 }
201
CreateCodecContext()202 Status VideoFfmpegDecoderPlugin::CreateCodecContext()
203 {
204 auto context = avcodec_alloc_context3(avCodec_.get());
205 if (context == nullptr) {
206 MEDIA_LOG_E("cannot allocate codec context");
207 return Status::ERROR_UNKNOWN;
208 }
209 avCodecContext_ = std::shared_ptr<AVCodecContext>(context, [](AVCodecContext* ptr) {
210 if (ptr != nullptr) {
211 if (ptr->extradata) {
212 av_free(ptr->extradata);
213 ptr->extradata = nullptr;
214 }
215 avcodec_free_context(&ptr);
216 }
217 });
218 MEDIA_LOG_I("CreateCodecContext success");
219 return Status::OK;
220 }
221
InitCodecContext()222 void VideoFfmpegDecoderPlugin::InitCodecContext()
223 {
224 avCodecContext_->codec_type = AVMEDIA_TYPE_VIDEO;
225 FindInParameterMapThenAssignLocked<int64_t>(Tag::MEDIA_BITRATE, avCodecContext_->bit_rate);
226 FindInParameterMapThenAssignLocked<std::uint32_t>(Tag::VIDEO_WIDTH, width_);
227 FindInParameterMapThenAssignLocked<std::uint32_t>(Tag::VIDEO_HEIGHT, height_);
228 FindInParameterMapThenAssignLocked<Plugin::VideoPixelFormat>(Tag::VIDEO_PIXEL_FORMAT, pixelFormat_);
229 MEDIA_LOG_D("bitRate: " PUBLIC_LOG_D64 ", width: " PUBLIC_LOG_U32 ", height: " PUBLIC_LOG_U32
230 ", pixelFormat: " PUBLIC_LOG_U32, avCodecContext_->bit_rate, width_, height_, pixelFormat_);
231 SetCodecExtraData();
232 // Reset coded_width/_height to prevent it being reused from last time when
233 // the codec is opened again, causing a mismatch and possible segfault/corruption.
234 avCodecContext_->coded_width = 0;
235 avCodecContext_->coded_height = 0;
236 avCodecContext_->workaround_bugs |= FF_BUG_AUTODETECT;
237 avCodecContext_->err_recognition = 1;
238 }
239
DeinitCodecContext()240 void VideoFfmpegDecoderPlugin::DeinitCodecContext()
241 {
242 if (avCodecContext_ == nullptr) {
243 return;
244 }
245 if (avCodecContext_->extradata) {
246 av_free(avCodecContext_->extradata);
247 avCodecContext_->extradata = nullptr;
248 }
249 avCodecContext_->extradata_size = 0;
250 avCodecContext_->opaque = nullptr;
251 avCodecContext_->width = 0;
252 avCodecContext_->height = 0;
253 avCodecContext_->coded_width = 0;
254 avCodecContext_->coded_height = 0;
255 avCodecContext_->time_base.den = 0;
256 avCodecContext_->time_base.num = 0;
257 avCodecContext_->ticks_per_frame = 0;
258 avCodecContext_->sample_aspect_ratio.num = 0;
259 avCodecContext_->sample_aspect_ratio.den = 0;
260 avCodecContext_->get_buffer2 = nullptr;
261 }
262
SetCodecExtraData()263 void VideoFfmpegDecoderPlugin::SetCodecExtraData()
264 {
265 auto iter = videoDecParams_.find(Tag::MEDIA_CODEC_CONFIG);
266 if (iter == videoDecParams_.end() || !iter->second.SameTypeWith(typeid(std::vector<uint8_t>))) {
267 return;
268 }
269 auto codecConfig = Plugin::AnyCast<std::vector<uint8_t>>(iter->second);
270 int configSize = codecConfig.size();
271 if (configSize > 0) {
272 auto allocSize = AlignUp(configSize + AV_INPUT_BUFFER_PADDING_SIZE, STRIDE_ALIGN);
273 avCodecContext_->extradata = static_cast<uint8_t*>(av_mallocz(allocSize));
274 (void)memcpy_s(avCodecContext_->extradata, configSize, codecConfig.data(), configSize);
275 avCodecContext_->extradata_size = configSize;
276 MEDIA_LOG_I("SetCodecExtraData success");
277 }
278 }
279
OpenCodecContext()280 Status VideoFfmpegDecoderPlugin::OpenCodecContext()
281 {
282 AVCodec* vdec = avcodec_find_decoder(avCodecContext_->codec_id);
283 if (vdec == nullptr) {
284 MEDIA_LOG_E("Codec: " PUBLIC_LOG_D32 " is not found", static_cast<int32_t>(avCodecContext_->codec_id));
285 DeinitCodecContext();
286 return Status::ERROR_INVALID_PARAMETER;
287 }
288 auto res = avcodec_open2(avCodecContext_.get(), avCodec_.get(), nullptr);
289 if (res != 0) {
290 MEDIA_LOG_E("avcodec open error " PUBLIC_LOG_S " when start decoder ", AVStrError(res).c_str());
291 DeinitCodecContext();
292 return Status::ERROR_UNKNOWN;
293 }
294 MEDIA_LOG_I("OpenCodecContext success");
295 return Status::OK;
296 }
297
CloseCodecContext()298 Status VideoFfmpegDecoderPlugin::CloseCodecContext()
299 {
300 Status ret = Status::OK;
301 if (avCodecContext_ != nullptr) {
302 auto res = avcodec_close(avCodecContext_.get());
303 if (res != 0) {
304 DeinitCodecContext();
305 MEDIA_LOG_E("avcodec close error " PUBLIC_LOG_S " when stop decoder", AVStrError(res).c_str());
306 ret = Status::ERROR_UNKNOWN;
307 }
308 avCodecContext_.reset();
309 }
310 return ret;
311 }
312
Prepare()313 Status VideoFfmpegDecoderPlugin::Prepare()
314 {
315 {
316 OSAL::ScopedLock l(avMutex_);
317 if (state_ != State::INITIALIZED && state_ != State::PREPARED) {
318 return Status::ERROR_WRONG_STATE;
319 }
320 if (CreateCodecContext() != Status::OK) {
321 MEDIA_LOG_E("Create codec context fail");
322 return Status::ERROR_UNKNOWN;
323 }
324 InitCodecContext();
325 #ifdef DUMP_RAW_DATA
326 dumpFd_ = std::fopen("./vdec_out.yuv", "wb");
327 #endif
328 state_ = State::PREPARED;
329 }
330 avPacket_ = std::shared_ptr<AVPacket>(av_packet_alloc(), [](AVPacket* ptr) {
331 av_packet_free(&ptr);
332 });
333 outBufferQ_.SetActive(true);
334 MEDIA_LOG_I("Prepare success");
335 return Status::OK;
336 }
337
ResetLocked()338 Status VideoFfmpegDecoderPlugin::ResetLocked()
339 {
340 videoDecParams_.clear();
341 avCodecContext_.reset();
342 outBufferQ_.Clear();
343 if (scaleData_[0] != nullptr) {
344 if (isAllocScaleData_) {
345 av_free(scaleData_[0]);
346 isAllocScaleData_ = false;
347 }
348 for (int32_t i = 0; i < AV_NUM_DATA_POINTERS; i++) {
349 scaleData_[i] = nullptr;
350 scaleLineSize_[i] = 0;
351 }
352 }
353 #ifdef DUMP_RAW_DATA
354 if (dumpFd_) {
355 std::fclose(dumpFd_);
356 dumpFd_ = nullptr;
357 }
358 #endif
359 state_ = State::INITIALIZED;
360 return Status::OK;
361 }
362
Reset()363 Status VideoFfmpegDecoderPlugin::Reset()
364 {
365 OSAL::ScopedLock l(avMutex_);
366 return ResetLocked();
367 }
368
Start()369 Status VideoFfmpegDecoderPlugin::Start()
370 {
371 {
372 OSAL::ScopedLock l(avMutex_);
373 if (state_ != State::PREPARED) {
374 return Status::ERROR_WRONG_STATE;
375 }
376 if (OpenCodecContext() != Status::OK) {
377 MEDIA_LOG_E("Open codec context fail");
378 return Status::ERROR_UNKNOWN;
379 }
380 state_ = State::RUNNING;
381 }
382 outBufferQ_.SetActive(true);
383 decodeTask_->Start();
384 MEDIA_LOG_I("Start success");
385 return Status::OK;
386 }
387
Stop()388 Status VideoFfmpegDecoderPlugin::Stop()
389 {
390 Status ret = Status::OK;
391 {
392 OSAL::ScopedLock l(avMutex_);
393 ret = CloseCodecContext();
394 #ifdef DUMP_RAW_DATA
395 if (dumpFd_) {
396 std::fclose(dumpFd_);
397 dumpFd_ = nullptr;
398 }
399 #endif
400 state_ = State::INITIALIZED;
401 }
402 outBufferQ_.SetActive(false);
403 decodeTask_->Stop();
404 MEDIA_LOG_I("Stop success");
405 return ret;
406 }
407
QueueOutputBuffer(const std::shared_ptr<Buffer> & outputBuffer,int32_t timeoutMs)408 Status VideoFfmpegDecoderPlugin::QueueOutputBuffer(const std::shared_ptr<Buffer>& outputBuffer, int32_t timeoutMs)
409 {
410 outBufferQ_.Push(outputBuffer);
411 MEDIA_LOG_DD("QueueOutputBuffer success");
412 return Status::OK;
413 }
414
Flush()415 Status VideoFfmpegDecoderPlugin::Flush()
416 {
417 OSAL::ScopedLock l(avMutex_);
418 if (avCodecContext_ != nullptr) {
419 // flush avcodec buffers
420 }
421 return Status::OK;
422 }
423
QueueInputBuffer(const std::shared_ptr<Buffer> & inputBuffer,int32_t timeoutMs)424 Status VideoFfmpegDecoderPlugin::QueueInputBuffer(const std::shared_ptr<Buffer>& inputBuffer, int32_t timeoutMs)
425 {
426 if (inputBuffer->IsEmpty() && !(inputBuffer->flag & BUFFER_FLAG_EOS)) {
427 MEDIA_LOG_E("decoder does not support fd buffer");
428 return Status::ERROR_INVALID_DATA;
429 }
430 Status ret = Status::OK;
431 {
432 OSAL::ScopedLock l(avMutex_);
433 ret = SendBufferLocked(inputBuffer);
434 }
435 NotifyInputBufferDone(inputBuffer);
436 MEDIA_LOG_DD("QueueInputBuffer ret: " PUBLIC_LOG_U32, ret);
437 return ret;
438 }
439
SendBufferLocked(const std::shared_ptr<Buffer> & inputBuffer)440 Status VideoFfmpegDecoderPlugin::SendBufferLocked(const std::shared_ptr<Buffer>& inputBuffer)
441 {
442 if (state_ != State::RUNNING) {
443 MEDIA_LOG_W("SendBufferLocked in wrong state: " PUBLIC_LOG_D32, state_);
444 return Status::ERROR_WRONG_STATE;
445 }
446 if (inputBuffer && !(inputBuffer->flag & BUFFER_FLAG_EOS)) {
447 auto inputMemory = inputBuffer->GetMemory();
448 const uint8_t* ptr = inputMemory->GetReadOnlyData();
449 auto bufferLength = inputMemory->GetSize();
450 size_t bufferEnd = bufferLength;
451 // pad to data if needed
452 if ((bufferLength % AV_INPUT_BUFFER_PADDING_SIZE != 0) &&
453 (bufferLength - bufferEnd + bufferLength % AV_INPUT_BUFFER_PADDING_SIZE < AV_INPUT_BUFFER_PADDING_SIZE)) {
454 if (paddedBufferSize_ < bufferLength + AV_INPUT_BUFFER_PADDING_SIZE) {
455 paddedBufferSize_ = bufferLength + AV_INPUT_BUFFER_PADDING_SIZE;
456 paddedBuffer_.reserve(paddedBufferSize_);
457 MEDIA_LOG_I("increase padded buffer size to " PUBLIC_LOG_ZU, paddedBufferSize_);
458 }
459 paddedBuffer_.assign(ptr, ptr + bufferLength);
460 paddedBuffer_.insert(paddedBuffer_.end(), AV_INPUT_BUFFER_PADDING_SIZE, 0);
461 ptr = paddedBuffer_.data();
462 }
463 avPacket_->data = const_cast<uint8_t*>(ptr);
464 avPacket_->size = static_cast<int32_t>(bufferLength);
465 avPacket_->pts = static_cast<int64_t>(inputBuffer->pts);
466 }
467 auto ret = avcodec_send_packet(avCodecContext_.get(), avPacket_.get());
468 av_packet_unref(avPacket_.get());
469 if (ret < 0) {
470 MEDIA_LOG_DD("send buffer error " PUBLIC_LOG_S, AVStrError(ret).c_str());
471 return Status::ERROR_NO_MEMORY;
472 }
473 return Status::OK;
474 }
475
476 #ifdef DUMP_RAW_DATA
DumpVideoRawOutData()477 void VideoFfmpegDecoderPlugin::DumpVideoRawOutData()
478 {
479 if (dumpFd_ == nullptr) {
480 return;
481 }
482 if (pixelFormat_ == VideoPixelFormat::YUV420P) {
483 if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
484 std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
485 scaleLineSize_[0] * height_, 1, dumpFd_);
486 }
487 if (scaleData_[1] != nullptr && scaleLineSize_[1] != 0) {
488 std::fwrite(reinterpret_cast<const char*>(scaleData_[1]),
489 scaleLineSize_[1] * height_ / 2, 1, dumpFd_); // 2
490 }
491 if (scaleData_[2] != nullptr && scaleLineSize_[2] != 0) { // 2
492 std::fwrite(reinterpret_cast<const char*>(scaleData_[2]),
493 scaleLineSize_[2] * height_ / 2, 1, dumpFd_); // 2
494 }
495 } else if (pixelFormat_ == VideoPixelFormat::NV21 || pixelFormat_ == VideoPixelFormat::NV12) {
496 if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
497 std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
498 scaleLineSize_[0] * height_, 1, dumpFd_);
499 }
500 if (scaleData_[1] != nullptr && scaleLineSize_[1] != 0) {
501 std::fwrite(reinterpret_cast<const char*>(scaleData_[1]),
502 scaleLineSize_[1] * height_ / 2, 1, dumpFd_); // 2
503 }
504 } else if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
505 pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
506 if (scaleData_[0] != nullptr && scaleLineSize_[0] != 0) {
507 std::fwrite(reinterpret_cast<const char*>(scaleData_[0]),
508 scaleLineSize_[0] * height_, 1, dumpFd_);
509 }
510 }
511 }
512 #endif
513
CreateSwsContext()514 Status VideoFfmpegDecoderPlugin::CreateSwsContext()
515 {
516 if (swsCtx_ != nullptr) {
517 return Status::OK;
518 }
519 auto swsContext = sws_getContext(cachedFrame_->width, cachedFrame_->height,
520 static_cast<enum AVPixelFormat>(cachedFrame_->format),
521 static_cast<int32_t>(width_), static_cast<int32_t>(height_),
522 ConvertPixelFormatToFFmpeg(pixelFormat_), SWS_BILINEAR, NULL, NULL, NULL);
523 FALSE_RETURN_V_MSG_E(swsContext != nullptr, Status::ERROR_UNKNOWN, "sws_getContext fail");
524 swsCtx_ = std::shared_ptr<struct SwsContext>(swsContext, [](struct SwsContext *ptr) {
525 if (ptr != nullptr) {
526 sws_freeContext(ptr);
527 }
528 });
529 FALSE_RETURN_V_MSG_E(swsCtx_ != nullptr, Status::ERROR_NO_MEMORY, "create swsCtx fail");
530 if (scaleData_[0] == nullptr && !isAllocScaleData_) {
531 auto ret = av_image_alloc(scaleData_, scaleLineSize_, width_, height_,
532 ConvertPixelFormatToFFmpeg(pixelFormat_), STRIDE_ALIGN);
533 FALSE_RETURN_V_MSG_E(ret >= 0, Status::ERROR_UNKNOWN, "av_image_fill_linesizes fail: " PUBLIC_LOG_D32, ret);
534 MEDIA_LOG_D("pixelFormat_: " PUBLIC_LOG_U32 ", pix_fmt: " PUBLIC_LOG_D32,
535 pixelFormat_, ConvertPixelFormatToFFmpeg(pixelFormat_));
536 for (int32_t i = 0; i < AV_NUM_DATA_POINTERS; i++) {
537 MEDIA_LOG_D("scaleData[" PUBLIC_LOG_D32 "]: " PUBLIC_LOG_P ", scaleLineSize[" PUBLIC_LOG_D32 "]: "
538 PUBLIC_LOG_D32, i, scaleData_[i], i, scaleLineSize_[i]);
539 if (scaleData_[i] && !scaleLineSize_[i]) {
540 MEDIA_LOG_E("scaleFrame is broken, i: " PUBLIC_LOG_D32, i);
541 return Status::ERROR_UNKNOWN;
542 }
543 }
544 isAllocScaleData_ = true;
545 }
546 MEDIA_LOG_D("CreateSwsContext success");
547 return Status::OK;
548 }
549
ScaleVideoFrame()550 Status VideoFfmpegDecoderPlugin::ScaleVideoFrame()
551 {
552 if (ConvertPixelFormatFromFFmpeg(static_cast<AVPixelFormat>(cachedFrame_->format)) == pixelFormat_ &&
553 static_cast<uint32_t>(cachedFrame_->width) == width_ &&
554 static_cast<uint32_t>(cachedFrame_->height) == height_) {
555 for (int32_t i = 0; cachedFrame_->linesize[i] > 0; i++) {
556 scaleData_[i] = cachedFrame_->data[i];
557 scaleLineSize_[i] = cachedFrame_->linesize[i];
558 }
559 return Status::OK;
560 }
561 auto ret = CreateSwsContext();
562 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "CreateSwsContext fail: " PUBLIC_LOG_D32, ret);
563 int32_t res = sws_scale(swsCtx_.get(), cachedFrame_->data, cachedFrame_->linesize, 0, cachedFrame_->height,
564 scaleData_, scaleLineSize_);
565 FALSE_RETURN_V_MSG_E(res >= 0, Status::ERROR_UNKNOWN, "sws_scale fail: " PUBLIC_LOG_D32, ret);
566 MEDIA_LOG_D("ScaleVideoFrame success");
567 return Status::OK;
568 }
569
570 #ifndef OHOS_LITE
WriteYuvDataStride(const std::shared_ptr<Buffer> & frameBuffer,int32_t stride)571 Status VideoFfmpegDecoderPlugin::WriteYuvDataStride(const std::shared_ptr<Buffer>& frameBuffer, int32_t stride)
572 {
573 auto frameBufferMem = frameBuffer->GetMemory();
574 size_t srcPos = 0;
575 size_t dstPos = 0;
576 if (pixelFormat_ == VideoPixelFormat::YUV420P) {
577 auto writeSize = scaleLineSize_[0];
578 for (uint32_t colNum = 0; colNum < height_; colNum++) {
579 frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
580 dstPos += stride;
581 }
582 srcPos = 0;
583 writeSize = scaleLineSize_[1];
584 for (uint32_t colNum = 0; colNum < height_; colNum++) {
585 frameBufferMem->Write(scaleData_[1] + srcPos, writeSize, dstPos);
586 dstPos += stride;
587 }
588 srcPos = 0;
589 writeSize = scaleLineSize_[2]; // 2
590 for (uint32_t colNum = 0; colNum < height_; colNum++) {
591 frameBufferMem->Write(scaleData_[2] + srcPos, writeSize, dstPos); // 2
592 dstPos += stride;
593 }
594 } else if ((pixelFormat_ == VideoPixelFormat::NV12) || (pixelFormat_ == VideoPixelFormat::NV21)) {
595 auto writeSize = scaleLineSize_[0];
596 for (uint32_t colNum = 0; colNum < height_; colNum++) {
597 frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
598 dstPos += stride;
599 }
600 srcPos = 0;
601 writeSize = scaleLineSize_[1];
602 for (uint32_t colNum = 0; colNum < height_; colNum++) {
603 frameBufferMem->Write(scaleData_[1] + srcPos, writeSize, dstPos);
604 dstPos += stride;
605 }
606 } else {
607 return Status::ERROR_UNSUPPORTED_FORMAT;
608 }
609 MEDIA_LOG_D("WriteYuvDataStride success");
610 return Status::OK;
611 }
612
WriteRgbDataStride(const std::shared_ptr<Buffer> & frameBuffer,int32_t stride)613 Status VideoFfmpegDecoderPlugin::WriteRgbDataStride(const std::shared_ptr<Buffer>& frameBuffer, int32_t stride)
614 {
615 auto frameBufferMem = frameBuffer->GetMemory();
616 if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
617 pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
618 size_t srcPos = 0;
619 size_t dstPos = 0;
620 auto writeSize = scaleLineSize_[0];
621 for (uint32_t colNum = 0; colNum < height_; colNum++) {
622 frameBufferMem->Write(scaleData_[0] + srcPos, writeSize, dstPos);
623 dstPos += stride;
624 }
625 } else {
626 return Status::ERROR_UNSUPPORTED_FORMAT;
627 }
628 MEDIA_LOG_D("WriteRgbDataStride success");
629 return Status::OK;
630 }
631 #endif
632
WriteYuvData(const std::shared_ptr<Buffer> & frameBuffer)633 Status VideoFfmpegDecoderPlugin::WriteYuvData(const std::shared_ptr<Buffer>& frameBuffer)
634 {
635 auto frameBufferMem = frameBuffer->GetMemory();
636 #ifndef OHOS_LITE
637 if (frameBufferMem->GetMemoryType() == Plugin::MemoryType::SURFACE_BUFFER) {
638 std::shared_ptr<Plugin::SurfaceMemory> surfaceMemory =
639 Plugin::ReinterpretPointerCast<Plugin::SurfaceMemory>(frameBufferMem);
640 auto stride = surfaceMemory->GetSurfaceBufferStride();
641 if (stride % width_) {
642 return WriteYuvDataStride(frameBuffer, stride);
643 }
644 }
645 #endif
646 size_t ySize = static_cast<size_t>(scaleLineSize_[0] * height_);
647 // AV_PIX_FMT_YUV420P: scaleLineSize_[0] = scaleLineSize_[1] * 2 = scaleLineSize_[2] * 2
648 // AV_PIX_FMT_NV12: scaleLineSize_[0] = scaleLineSize_[1]
649 size_t uvSize = static_cast<size_t>(scaleLineSize_[1] * height_ / 2); // 2
650 size_t frameSize = 0;
651 if (pixelFormat_ == VideoPixelFormat::YUV420P) {
652 frameSize = ySize + (uvSize * 2); // 2
653 } else if (pixelFormat_ == VideoPixelFormat::NV21 || pixelFormat_ == VideoPixelFormat::NV12) {
654 frameSize = ySize + uvSize;
655 }
656 FALSE_RETURN_V_MSG_E(frameBufferMem->GetCapacity() >= frameSize, Status::ERROR_NO_MEMORY,
657 "output buffer size is not enough: real[" PUBLIC_LOG "zu], need[" PUBLIC_LOG "zu]",
658 frameBufferMem->GetCapacity(), frameSize);
659 if (pixelFormat_ == VideoPixelFormat::YUV420P) {
660 frameBufferMem->Write(scaleData_[0], ySize);
661 frameBufferMem->Write(scaleData_[1], uvSize);
662 frameBufferMem->Write(scaleData_[2], uvSize); // 2
663 } else if ((pixelFormat_ == VideoPixelFormat::NV12) || (pixelFormat_ == VideoPixelFormat::NV21)) {
664 frameBufferMem->Write(scaleData_[0], ySize);
665 frameBufferMem->Write(scaleData_[1], uvSize);
666 } else {
667 return Status::ERROR_UNSUPPORTED_FORMAT;
668 }
669 MEDIA_LOG_DD("WriteYuvData success");
670 return Status::OK;
671 }
672
WriteRgbData(const std::shared_ptr<Buffer> & frameBuffer)673 Status VideoFfmpegDecoderPlugin::WriteRgbData(const std::shared_ptr<Buffer>& frameBuffer)
674 {
675 auto frameBufferMem = frameBuffer->GetMemory();
676 #ifndef OHOS_LITE
677 if (frameBufferMem->GetMemoryType() == Plugin::MemoryType::SURFACE_BUFFER) {
678 std::shared_ptr<Plugin::SurfaceMemory> surfaceMemory =
679 Plugin::ReinterpretPointerCast<Plugin::SurfaceMemory>(frameBufferMem);
680 auto stride = surfaceMemory->GetSurfaceBufferStride();
681 if (stride % width_) {
682 return WriteRgbDataStride(frameBuffer, stride);
683 }
684 }
685 #endif
686 size_t frameSize = static_cast<size_t>(scaleLineSize_[0] * height_);
687 FALSE_RETURN_V_MSG_E(frameBufferMem->GetCapacity() >= frameSize, Status::ERROR_NO_MEMORY,
688 "output buffer size is not enough: real[" PUBLIC_LOG "zu], need[" PUBLIC_LOG "zu]",
689 frameBufferMem->GetCapacity(), frameSize);
690 if (pixelFormat_ == VideoPixelFormat::RGBA || pixelFormat_ == VideoPixelFormat::ARGB ||
691 pixelFormat_ == VideoPixelFormat::ABGR || pixelFormat_ == VideoPixelFormat::BGRA) {
692 frameBufferMem->Write(scaleData_[0], frameSize);
693 } else {
694 return Status::ERROR_UNSUPPORTED_FORMAT;
695 }
696 MEDIA_LOG_D("WriteRgbData success");
697 return Status::OK;
698 }
699
FillFrameBuffer(const std::shared_ptr<Buffer> & frameBuffer)700 Status VideoFfmpegDecoderPlugin::FillFrameBuffer(const std::shared_ptr<Buffer>& frameBuffer)
701 {
702 MEDIA_LOG_DD("receive one frame: " PUBLIC_LOG_D32 ", picture type: " PUBLIC_LOG_D32 ", pixel format: "
703 PUBLIC_LOG_D32 ", packet size: " PUBLIC_LOG_D32, cachedFrame_->key_frame,
704 static_cast<int32_t>(cachedFrame_->pict_type), cachedFrame_->format, cachedFrame_->pkt_size);
705 FALSE_RETURN_V_MSG_E((cachedFrame_->flags & AV_FRAME_FLAG_CORRUPT) == 0, Status::ERROR_INVALID_DATA,
706 "decoded frame is corrupt");
707 auto ret = ScaleVideoFrame();
708 FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "ScaleVideoFrame fail: " PUBLIC_LOG_D32, ret);
709 auto bufferMeta = frameBuffer->GetBufferMeta();
710 if (bufferMeta != nullptr && bufferMeta->GetType() == BufferMetaType::VIDEO) {
711 std::shared_ptr<VideoBufferMeta> videoMeta = ReinterpretPointerCast<VideoBufferMeta>(bufferMeta);
712 videoMeta->videoPixelFormat = pixelFormat_;
713 videoMeta->height = height_;
714 videoMeta->width = width_;
715 for (int i = 0; scaleLineSize_[i] > 0; ++i) {
716 videoMeta->stride.emplace_back(scaleLineSize_[i]);
717 }
718 videoMeta->planes = videoMeta->stride.size();
719 }
720 #ifdef DUMP_RAW_DATA
721 DumpVideoRawOutData();
722 #endif
723 auto newFormat = ConvertPixelFormatToFFmpeg(pixelFormat_);
724 if (IsYuvFormat(newFormat)) {
725 FALSE_RETURN_V_MSG_E(WriteYuvData(frameBuffer) == Status::OK, Status::ERROR_UNSUPPORTED_FORMAT,
726 "Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
727 } else if (IsRgbFormat(newFormat)) {
728 FALSE_RETURN_V_MSG_E(WriteRgbData(frameBuffer) == Status::OK, Status::ERROR_UNSUPPORTED_FORMAT,
729 "Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
730 } else {
731 MEDIA_LOG_E("Unsupported pixel format: " PUBLIC_LOG_U32, pixelFormat_);
732 return Status::ERROR_UNSUPPORTED_FORMAT;
733 }
734 frameBuffer->pts = static_cast<uint64_t>(cachedFrame_->pts);
735 MEDIA_LOG_DD("FillFrameBuffer success");
736 return Status::OK;
737 }
738
ReceiveBufferLocked(const std::shared_ptr<Buffer> & frameBuffer)739 Status VideoFfmpegDecoderPlugin::ReceiveBufferLocked(const std::shared_ptr<Buffer>& frameBuffer)
740 {
741 if (state_ != State::RUNNING) {
742 MEDIA_LOG_W("ReceiveBufferLocked in wrong state: " PUBLIC_LOG_D32, state_);
743 return Status::ERROR_WRONG_STATE;
744 }
745 Status status;
746 auto ret = avcodec_receive_frame(avCodecContext_.get(), cachedFrame_.get());
747 if (ret >= 0) {
748 status = FillFrameBuffer(frameBuffer);
749 } else if (ret == AVERROR_EOF) {
750 MEDIA_LOG_I("eos received");
751 frameBuffer->GetMemory()->Reset();
752 frameBuffer->flag |= BUFFER_FLAG_EOS;
753 avcodec_flush_buffers(avCodecContext_.get());
754 status = Status::END_OF_STREAM;
755 } else {
756 MEDIA_LOG_DD("video decoder receive error: " PUBLIC_LOG_S, AVStrError(ret).c_str());
757 status = Status::ERROR_TIMED_OUT;
758 }
759 av_frame_unref(cachedFrame_.get());
760 MEDIA_LOG_DD("ReceiveBufferLocked status: " PUBLIC_LOG_U32, status);
761 return status;
762 }
763
ReceiveFrameBuffer()764 void VideoFfmpegDecoderPlugin::ReceiveFrameBuffer()
765 {
766 std::shared_ptr<Buffer> frameBuffer = outBufferQ_.Pop();
767 if (frameBuffer == nullptr || frameBuffer->IsEmpty()) {
768 MEDIA_LOG_W("cannot fetch valid buffer to output");
769 return;
770 }
771 auto frameMeta = frameBuffer->GetBufferMeta();
772 if (frameMeta == nullptr || frameMeta->GetType() != BufferMetaType::VIDEO) {
773 MEDIA_LOG_W("output buffer is not video buffer");
774 return;
775 }
776 Status status;
777 {
778 OSAL::ScopedLock l(avMutex_);
779 status = ReceiveBufferLocked(frameBuffer);
780 }
781 if (status == Status::OK || status == Status::END_OF_STREAM) {
782 NotifyOutputBufferDone(frameBuffer);
783 } else {
784 outBufferQ_.Push(frameBuffer);
785 }
786 }
787
NotifyInputBufferDone(const std::shared_ptr<Buffer> & input)788 void VideoFfmpegDecoderPlugin::NotifyInputBufferDone(const std::shared_ptr<Buffer>& input)
789 {
790 if (dataCb_ != nullptr) {
791 dataCb_->OnInputBufferDone(input);
792 }
793 }
794
NotifyOutputBufferDone(const std::shared_ptr<Buffer> & output)795 void VideoFfmpegDecoderPlugin::NotifyOutputBufferDone(const std::shared_ptr<Buffer>& output)
796 {
797 if (dataCb_ != nullptr) {
798 dataCb_->OnOutputBufferDone(output);
799 }
800 }
801
GetAllocator()802 std::shared_ptr<Allocator> VideoFfmpegDecoderPlugin::GetAllocator()
803 {
804 return nullptr;
805 }
806 } // namespace Plugin
807 } // namespace Media
808 } // namespace OHOS
809 #endif
810