1 /*
2 * Copyright (C) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "media_log.h"
17 #include "codec/video/encoder/video_encoder_engine_impl.h"
18 #include "codec/common/codec_common.h"
19 #include <native_avcodec_audioencoder.h>
20 #include <native_avcodec_videoencoder.h>
21 #include <native_avcodec_videodecoder.h>
22 #include "render/graphics/graphics_render_engine.h"
23 #include "securec.h"
24
25 namespace OHOS {
26 namespace Media {
27
28 namespace {
29 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN_VIDEOEDITOR, "VideoEditorEncode"};
30 }
31
VideoEncoderEngineImpl(uint64_t id,VideoEncodeCallback * cb)32 VideoEncoderEngineImpl::VideoEncoderEngineImpl(uint64_t id, VideoEncodeCallback* cb)
33 : id_(id), cb_(cb)
34 {
35 logTag_ = "video-encoder-engine-" + std::to_string(id_);
36 }
37
~VideoEncoderEngineImpl()38 VideoEncoderEngineImpl::~VideoEncoderEngineImpl()
39 {
40 MEDIA_LOGD("[%{public}s] destruct.", logTag_.c_str());
41 }
42
GetId() const43 uint64_t VideoEncoderEngineImpl::GetId() const
44 {
45 return id_;
46 }
47
InitVideoMuxer(const VideoMuxerParam & muxerParam)48 VEFError VideoEncoderEngineImpl::InitVideoMuxer(const VideoMuxerParam& muxerParam)
49 {
50 auto muxer = std::make_shared<VideoMuxer>(id_);
51 VEFError error = muxer->Init(muxerParam);
52 if (error != VEFError::ERR_OK) {
53 MEDIA_LOGE("[%{public}s] init video muxer failed, error: %{public}d.", logTag_.c_str(), error);
54 UnInit();
55 return error;
56 }
57 muxer_ = muxer;
58 return VEFError::ERR_OK;
59 }
60
InitAudioStreamEncoder(OH_AVFormat * audioFormat)61 VEFError VideoEncoderEngineImpl::InitAudioStreamEncoder(OH_AVFormat* audioFormat)
62 {
63 if (audioFormat == nullptr) {
64 MEDIA_LOGW("[%{public}s] audio encoder disable.", logTag_.c_str());
65 audioEncoderState_ = CodecState::FINISH_SUCCESS;
66 OnEncodeResult(CodecResult::SUCCESS);
67 return VEFError::ERR_OK;
68 }
69
70 std::function onDataOutput = [&](OH_AVCodec* codec, uint32_t index, OH_AVMemory* data, OH_AVCodecBufferAttr* attr)
71 -> VEFError {
72 return OnAudioEncodeOutput(codec, index, data, attr);
73 };
74
75 audioEncoder_ = std::make_shared<AudioEncoder>(id_, onDataOutput);
76 auto error = muxer_->AddAudioTrack(audioFormat);
77 if (error != VEFError::ERR_OK) {
78 MEDIA_LOGE("[%{public}s] add audio track to muxer failed with error: %{public}d.", logTag_.c_str(), error);
79 UnInit();
80 return error;
81 }
82 OH_AVFormat* format = OH_AVFormat_Create();
83 if (!format) {
84 MEDIA_LOGE("[%{public}s] OH_AVFormat_Create failed.", logTag_.c_str());
85 return VEFError::ERR_OOM;
86 }
87
88 OH_AVFormat_Copy(format, audioFormat);
89
90 // 由于OHNative支持的采样格式有限，优先采样SAMPLE_S16LE，不同采样格式内部会实现转换
91 OH_AVFormat_SetIntValue(format, OH_MD_KEY_AUDIO_SAMPLE_FORMAT, SAMPLE_S16LE);
92
93 error = audioEncoder_->Init(format);
94 if (error != VEFError::ERR_OK) {
95 MEDIA_LOGE("[%{public}s] init audio encoder failed with error: %{public}d.", logTag_.c_str(), error);
96 audioEncoder_ = nullptr;
97 OH_AVFormat_Destroy(format);
98 return error;
99 }
100
101 MEDIA_LOGI("[%{public}s] init audio success.", logTag_.c_str());
102 return VEFError::ERR_OK;
103 }
104
InitVideoStreamEncoder(OH_AVFormat * videoFormat)105 VEFError VideoEncoderEngineImpl::InitVideoStreamEncoder(OH_AVFormat* videoFormat)
106 {
107 MEDIA_LOGI("[%{public}s] init.", logTag_.c_str());
108 if (videoFormat == nullptr) {
109 MEDIA_LOGE("[%{public}s] init failed, the parameter videoFormat is nullptr.", logTag_.c_str());
110 return VEFError::ERR_INTERNAL_ERROR;
111 }
112 std::function onOutputData = [&](OH_AVMemory* data, OH_AVCodecBufferAttr* attr) {
113 OnVideoNewOutputDataCallBack(data, attr);
114 };
115
116 encoder_ = std::make_shared<VideoEncoder>(id_, onOutputData);
117 if (encoder_ == nullptr) {
118 MEDIA_LOGE("[%{public}s] create video encoder failed.", logTag_.c_str());
119 UnInit();
120 return VEFError::ERR_INTERNAL_ERROR;
121 }
122 VEFError error = encoder_->Init(videoFormat);
123 if (error != VEFError::ERR_OK) {
124 MEDIA_LOGE("[%{public}s] init video encoder object failed with error: %{public}d.", logTag_.c_str(), error);
125 UnInit();
126 return error;
127 }
128 MEDIA_LOGI("[%{public}s] init video stream encoder object success.", logTag_.c_str());
129
130 error = muxer_->AddVideoTrack(videoFormat);
131 if (error != VEFError::ERR_OK) {
132 MEDIA_LOGE("[%{public}s] add video track to muxer failed with error: %{public}d.", logTag_.c_str(), error);
133 UnInit();
134 return error;
135 }
136
137 MEDIA_LOGI("[%{public}s] init video stream encoder success.", logTag_.c_str());
138 return VEFError::ERR_OK;
139 }
140
Init(const VideoEncodeParam & encodeParam)141 VEFError VideoEncoderEngineImpl::Init(const VideoEncodeParam& encodeParam)
142 {
143 MEDIA_LOGI("[%{public}s] init.", logTag_.c_str());
144 VEFError error = InitVideoMuxer(encodeParam.muxerParam);
145 if (error != VEFError::ERR_OK) {
146 MEDIA_LOGE("[%{public}s] init video file muxer failed, error: %{public}d.", logTag_.c_str(), error);
147 return error;
148 }
149
150 error = InitVideoStreamEncoder(encodeParam.videoTrunkFormat);
151 if (error != VEFError::ERR_OK) {
152 MEDIA_LOGE("[%{public}s] init video stream encoder failed, error: %{public}d.", logTag_.c_str(), error);
153 return error;
154 }
155
156 return InitAudioStreamEncoder(encodeParam.audioTrunkFormat);
157 }
158
UnInit()159 void VideoEncoderEngineImpl::UnInit()
160 {
161 MEDIA_LOGI("[%{public}s] uninit.", logTag_.c_str());
162 if (encoder_ != nullptr) {
163 MEDIA_LOGI("[%{public}s] uninit encoder.", logTag_.c_str());
164 encoder_ = nullptr;
165 }
166 if (audioEncoder_ != nullptr) {
167 MEDIA_LOGI("[%{public}s] uninit audio encoder.", logTag_.c_str());
168 audioEncoder_ = nullptr;
169 }
170 if (muxer_ != nullptr) {
171 MEDIA_LOGI("[%{public}s] uninit muxer.", logTag_.c_str());
172 muxer_ = nullptr;
173 }
174 MEDIA_LOGI("[%{public}s] uninit finish.", logTag_.c_str());
175 }
176
GetVideoInputWindow()177 OHNativeWindow* VideoEncoderEngineImpl::GetVideoInputWindow()
178 {
179 return encoder_->GetOHNativeWindow();
180 }
181
GetAudioInputBufferQueue() const182 std::shared_ptr<PcmBufferQueue> VideoEncoderEngineImpl::GetAudioInputBufferQueue() const
183 {
184 return audioEncoder_ == nullptr ? nullptr : audioEncoder_->GetPcmInputBufferQueue();
185 }
186
StartEncode()187 VEFError VideoEncoderEngineImpl::StartEncode()
188 {
189 MEDIA_LOGI("[%{public}s] start encode.", logTag_.c_str());
190 VEFError error = muxer_->Start();
191 if (error != VEFError::ERR_OK) {
192 MEDIA_LOGE("[%{public}s] start muxer failed with error: %{public}d.", logTag_.c_str(), error);
193 return error;
194 }
195 error = encoder_->Start();
196 if (error != VEFError::ERR_OK) {
197 MEDIA_LOGE("[%{public}s] start encoder failed with error: %{public}d.", logTag_.c_str(), error);
198 return error;
199 }
200 videoEncoderState_ = CodecState::RUNNING;
201 if (audioEncoder_ == nullptr) {
202 videoEncoderState_ = CodecState::FINISH_SUCCESS;
203 MEDIA_LOGE("[%{public}s] startEncode audioEncoder_ is null.", logTag_.c_str());
204 } else {
205 VEFError error = audioEncoder_->Start();
206 if (error != VEFError::ERR_OK) {
207 MEDIA_LOGE("[%{public}s] start audio encoder failed with error: %{public}d.", logTag_.c_str(), error);
208 return error;
209 }
210 audioEncoderState_ = CodecState::RUNNING;
211 }
212 MEDIA_LOGI("[%{public}s] start encoder engine success.", logTag_.c_str());
213 return VEFError::ERR_OK;
214 }
215
StopEncode()216 VEFError VideoEncoderEngineImpl::StopEncode()
217 {
218 MEDIA_LOGI("[%{public}s] stop encode.", logTag_.c_str());
219 if (audioEncoder_ != nullptr) {
220 VEFError error = audioEncoder_->Stop();
221 if (error != VEFError::ERR_OK) {
222 MEDIA_LOGE("[%{public}s] stop audio encoder failed with error: %{public}d.", logTag_.c_str(), error);
223 return error;
224 }
225 audioEncoderState_ = CodecState::CANCEL;
226 }
227
228 VEFError error = encoder_->Stop();
229 if (error != VEFError::ERR_OK) {
230 audioEncoderState_ = CodecState::FINISH_FAILED;
231 MEDIA_LOGE("[%{public}s] stop encoder failed with error: %{public}d.", logTag_.c_str(), error);
232 return error;
233 }
234 videoEncoderState_ = CodecState::CANCEL;
235 error = muxer_->Stop();
236 if (error != VEFError::ERR_OK) {
237 audioEncoderState_ = CodecState::FINISH_FAILED;
238 MEDIA_LOGE("[%{public}s] stop muxer failed with error: %{public}d.", logTag_.c_str(), error);
239 return error;
240 }
241 MEDIA_LOGI("[%{public}s] stop encode finish.", logTag_.c_str());
242 return VEFError::ERR_OK;
243 }
244
OnEncodeResult(CodecResult result)245 void VideoEncoderEngineImpl::OnEncodeResult(CodecResult result)
246 {
247 if (videoEncoderState_ == CodecState::FINISH_SUCCESS && audioEncoderState_ == CodecState::FINISH_SUCCESS) {
248 cb_->OnEncodeResult(result);
249 }
250 }
251
OnVideoNewOutputDataCallBack(OH_AVMemory * data,OH_AVCodecBufferAttr * attr)252 void VideoEncoderEngineImpl::OnVideoNewOutputDataCallBack(OH_AVMemory* data, OH_AVCodecBufferAttr* attr)
253 {
254 if (data == nullptr || attr == nullptr) {
255 MEDIA_LOGE("[%{public}s] invalid parameter, data or attr is nullptr.", logTag_.c_str());
256 return;
257 }
258 MEDIA_LOGD("[%{public}s] OnVideoNewOutputDataCallBack packet pts: %{public}" PRIu64 " flag: %{public}u",
259 logTag_.c_str(), attr->pts, attr->flags);
260 if (muxer_ == nullptr) {
261 MEDIA_LOGE("[%{public}s] muxer_ is nullptr.", logTag_.c_str());
262 return;
263 }
264 VEFError error = muxer_->WriteVideoData(data, attr);
265 if (error != VEFError::ERR_OK) {
266 MEDIA_LOGE("[%{public}s] WriteVideoData with error: %{public}d.", logTag_.c_str(), error);
267 return;
268 }
269 if (cb_ == nullptr) {
270 audioEncoderState_ = CodecState::FINISH_FAILED;
271 MEDIA_LOGE("[%{public}s] OnVideoNewOutputDataCallBack, but cb is expired.", logTag_.c_str());
272 return;
273 }
274 cb_->OnEncodeFrame(attr->pts);
275 if ((attr->flags & AVCODEC_BUFFER_FLAGS_EOS) == AVCODEC_BUFFER_FLAGS_EOS) {
276 MEDIA_LOGI("[%{public}s] encode video output eos.", logTag_.c_str());
277 videoEncoderState_ = CodecState::FINISH_SUCCESS;
278 OnEncodeResult(CodecResult::SUCCESS);
279 }
280 }
281
SendEos()282 VEFError VideoEncoderEngineImpl::SendEos()
283 {
284 MEDIA_LOGD("[%{public}s] send eos to encoder.", logTag_.c_str());
285 if (encoder_ == nullptr) {
286 MEDIA_LOGE("[%{public}s] encoder_ is nullptr.", logTag_.c_str());
287 return VEFError::ERR_INTERNAL_ERROR;
288 }
289 VEFError error = encoder_ ->SendEos();
290 if (error != VEFError::ERR_OK) {
291 MEDIA_LOGI("[%{public}s] send eos to video encoder failed, error: %{public}d.", logTag_.c_str(), error);
292 return error;
293 }
294 return VEFError::ERR_OK;
295 }
296
Flush()297 VEFError VideoEncoderEngineImpl::Flush()
298 {
299 MEDIA_LOGI("[%{public}s] finish encode.", logTag_.c_str());
300 if (audioEncoder_ != nullptr) {
301 VEFError error = audioEncoder_->Flush();
302 if (error != VEFError::ERR_OK) {
303 MEDIA_LOGE("[%{public}s] finish audio encoder failed with error: %{public}d.", logTag_.c_str(), error);
304 return error;
305 }
306 }
307 if (encoder_ == nullptr) {
308 MEDIA_LOGE("[%{public}s] encoder_ is nullptr.", logTag_.c_str());
309 return VEFError::ERR_INTERNAL_ERROR;
310 }
311 VEFError error = encoder_->Flush();
312 if (error != VEFError::ERR_OK) {
313 MEDIA_LOGE("[%{public}s] finish video encoder failed with error: %{public}d.", logTag_.c_str(), error);
314 return error;
315 }
316 return VEFError::ERR_OK;
317 }
318
OnAudioEncodeOutput(OH_AVCodec * codec,uint32_t index,OH_AVMemory * data,OH_AVCodecBufferAttr * attr)319 VEFError VideoEncoderEngineImpl::OnAudioEncodeOutput(OH_AVCodec *codec, uint32_t index, OH_AVMemory* data,
320 OH_AVCodecBufferAttr* attr)
321 {
322 if (codec == nullptr || data == nullptr || attr == nullptr) {
323 MEDIA_LOGE("[%{public}s] invalid parameter, codec or data or attr is nullptr.", logTag_.c_str());
324 audioEncoderState_ = CodecState::FINISH_FAILED;
325 return VEFError::ERR_INTERNAL_ERROR;
326 }
327 if (encoder_ == nullptr) {
328 MEDIA_LOGE("[%{public}s] encoder_ is nullptr.", logTag_.c_str());
329 return VEFError::ERR_INTERNAL_ERROR;
330 }
331 VEFError error = muxer_->WriteAudioData(data, attr);
332 if (error != VEFError::ERR_OK) {
333 MEDIA_LOGE("[%{public}s] WriteAudioData with error: %{public}d.", logTag_.c_str(), error);
334 return error;
335 }
336 OH_AVErrCode err = OH_AudioEncoder_FreeOutputData(codec, index);
337 if (err != AV_ERR_OK) {
338 MEDIA_LOGE("[%{public}s] OH_AudioEncoder_FreeOutputData with error: %{public}d.", logTag_.c_str(), err);
339 return VEFError::ERR_INTERNAL_ERROR;
340 }
341 if ((attr->flags & AVCODEC_BUFFER_FLAGS_EOS) == AVCODEC_BUFFER_FLAGS_EOS) {
342 MEDIA_LOGI("[%{public}s] audio encode output eos.", logTag_.c_str());
343 audioEncoderState_ = CodecState::FINISH_SUCCESS;
344 OnEncodeResult(CodecResult::SUCCESS);
345 }
346 return VEFError::ERR_OK;
347 }
348
349 } // namespace Media
350 } // namespace OHOS
351