1 /*
2 * Copyright (c) 2024-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "video_encoder.h"
17 #include "sample_callback.h"
18 #include "camera_log.h"
19 #include <sync_fence.h>
20 #include "native_window.h"
21
22 namespace OHOS {
23 namespace CameraStandard {
24
~VideoEncoder()25 VideoEncoder::~VideoEncoder()
26 {
27 MEDIA_INFO_LOG("~VideoEncoder enter");
28 if (codecSurface_) {
29 MEDIA_INFO_LOG("codecSurface refCount %{public}d", codecSurface_->GetSptrRefCount());
30 }
31 Release();
32 }
33
VideoEncoder(VideoCodecType type)34 VideoEncoder::VideoEncoder(VideoCodecType type) : videoCodecType_(type)
35 {
36 rotation_ = 0;
37 MEDIA_INFO_LOG("VideoEncoder enter");
38 }
39
Create(const std::string & codecMime)40 int32_t VideoEncoder::Create(const std::string &codecMime)
41 {
42 std::lock_guard<std::mutex> lock(encoderMutex_);
43 encoder_ = OH_VideoEncoder_CreateByMime(codecMime.data());
44 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Create failed");
45 return 0;
46 }
47
Config()48 int32_t VideoEncoder::Config()
49 {
50 std::lock_guard<std::mutex> lock(encoderMutex_);
51 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
52 std::unique_lock<std::mutex> contextLock(contextMutex_);
53 context_ = new CodecUserData;
54 // Configure video encoder
55 int32_t ret = Configure();
56 CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Configure failed");
57 // SetCallback for video encoder
58 ret = SetCallback(context_);
59 CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Set callback failed");
60 contextLock.unlock();
61 return 0;
62 }
63
Start()64 int32_t VideoEncoder::Start()
65 {
66 std::lock_guard<std::mutex> lock(encoderMutex_);
67 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
68 // Prepare video encoder
69 int ret = OH_VideoEncoder_Prepare(encoder_);
70 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Prepare failed, ret: %{public}d", ret);
71 // Start video encoder
72 ret = OH_VideoEncoder_Start(encoder_);
73 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Start failed, ret: %{public}d", ret);
74 isStarted_ = true;
75 return 0;
76 }
77
GetSurface()78 int32_t VideoEncoder::GetSurface()
79 {
80 std::lock_guard<std::mutex> lock(encoderMutex_);
81 OHNativeWindow *nativeWindow;
82 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
83 int ret = OH_VideoEncoder_GetSurface(encoder_, &nativeWindow);
84 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Get surface failed, ret: %{public}d", ret);
85 surfaceMutex_.lock();
86 codecSurface_ = nativeWindow->surface;
87 OH_NativeWindow_DestroyNativeWindow(nativeWindow);
88 CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, 1, "Surface is null");
89 surfaceMutex_.unlock();
90 return 0;
91 }
92
ReleaseSurfaceBuffer(sptr<FrameRecord> frameRecord)93 int32_t VideoEncoder::ReleaseSurfaceBuffer(sptr<FrameRecord> frameRecord)
94 {
95 CAMERA_SYNC_TRACE;
96 CHECK_AND_RETURN_RET_LOG(frameRecord->GetSurfaceBuffer() != nullptr, 1,
97 "SurfaceBuffer is released %{public}s", frameRecord->GetFrameId().c_str());
98 sptr<SyncFence> syncFence = SyncFence::INVALID_FENCE;
99 BufferRequestConfig requestConfig = {
100 .width = frameRecord->GetSurfaceBuffer()->GetWidth(),
101 .height = frameRecord->GetSurfaceBuffer()->GetHeight(),
102 .strideAlignment = 0x8, // default stride is 8 Bytes.
103 .format = frameRecord->GetSurfaceBuffer()->GetFormat(),
104 .usage = frameRecord->GetSurfaceBuffer()->GetUsage(),
105 .timeout = 0,
106 };
107 sptr<SurfaceBuffer> releaseBuffer;
108 {
109 std::lock_guard<std::mutex> lock(surfaceMutex_);
110 CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, 1, "codecSurface_ is null");
111 SurfaceError ret = codecSurface_->RequestBuffer(releaseBuffer, syncFence, requestConfig);
112 if (ret != SURFACE_ERROR_OK) {
113 MEDIA_ERR_LOG("RequestBuffer failed. %{public}d", ret);
114 return ret;
115 }
116 constexpr uint32_t waitForEver = -1;
117 (void)syncFence->Wait(waitForEver);
118
119 if (!releaseBuffer) {
120 MEDIA_ERR_LOG("Failed to requestBuffer, %{public}s", frameRecord->GetFrameId().c_str());
121 return ret;
122 }
123 ret = codecSurface_->DetachBufferFromQueue(releaseBuffer);
124 if (ret != SURFACE_ERROR_OK) {
125 MEDIA_ERR_LOG("Failed to detach buffer");
126 return ret;
127 }
128 }
129 frameRecord->SetSurfaceBuffer(releaseBuffer);
130 // after request surfaceBuffer
131 frameRecord->NotifyBufferRelease();
132 MEDIA_INFO_LOG("release codec surface buffer end");
133 return 0;
134 }
135
PushInputData(sptr<CodecAVBufferInfo> info)136 int32_t VideoEncoder::PushInputData(sptr<CodecAVBufferInfo> info)
137 {
138 std::lock_guard<std::mutex> lock(encoderMutex_);
139 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Decoder is null");
140 int32_t ret = AV_ERR_OK;
141 ret = OH_AVBuffer_SetBufferAttr(info->buffer, &info->attr);
142 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set avbuffer attr failed, ret: %{public}d", ret);
143 ret = OH_VideoEncoder_PushInputBuffer(encoder_, info->bufferIndex);
144 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Push input data failed, ret: %{public}d", ret);
145 return 0;
146 }
147
NotifyEndOfStream()148 int32_t VideoEncoder::NotifyEndOfStream()
149 {
150 std::lock_guard<std::mutex> lock(encoderMutex_);
151 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
152 int32_t ret = OH_VideoEncoder_NotifyEndOfStream(encoder_);
153 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1,
154 "Notify end of stream failed, ret: %{public}d", ret);
155 return 0;
156 }
157
FreeOutputData(uint32_t bufferIndex)158 int32_t VideoEncoder::FreeOutputData(uint32_t bufferIndex)
159 {
160 std::lock_guard<std::mutex> lock(encoderMutex_);
161 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
162 int32_t ret = OH_VideoEncoder_FreeOutputBuffer(encoder_, bufferIndex);
163 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1,
164 "Free output data failed, ret: %{public}d", ret);
165 return 0;
166 }
167
Stop()168 int32_t VideoEncoder::Stop()
169 {
170 CAMERA_SYNC_TRACE;
171 std::lock_guard<std::mutex> lock(encoderMutex_);
172 CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null");
173 int ret = OH_VideoEncoder_Stop(encoder_);
174 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Stop failed, ret: %{public}d", ret);
175 isStarted_ = false;
176 return 0;
177 }
178
RestartVideoCodec(shared_ptr<Size> size,int32_t rotation)179 void VideoEncoder::RestartVideoCodec(shared_ptr<Size> size, int32_t rotation)
180 {
181 Release();
182 size_ = size;
183 rotation_ = rotation;
184 MEDIA_INFO_LOG("VideoEncoder videoCodecType_ = %{public}d", videoCodecType_);
185 if (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_AVC) {
186 Create(MIME_VIDEO_AVC.data());
187 } else if (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_HEVC) {
188 Create(MIME_VIDEO_HEVC.data());
189 }
190 Config();
191 GetSurface();
192 Start();
193 }
194
EnqueueBuffer(sptr<FrameRecord> frameRecord,int32_t keyFrameInterval)195 bool VideoEncoder::EnqueueBuffer(sptr<FrameRecord> frameRecord, int32_t keyFrameInterval)
196 {
197 if (!isStarted_ || encoder_ == nullptr || size_ == nullptr) {
198 RestartVideoCodec(frameRecord->GetFrameSize(), frameRecord->GetRotation());
199 }
200 if (keyFrameInterval == KEY_FRAME_INTERVAL) {
201 std::lock_guard<std::mutex> lock(encoderMutex_);
202 OH_AVFormat *format = OH_AVFormat_Create();
203 OH_AVFormat_SetIntValue(format, OH_MD_KEY_REQUEST_I_FRAME, true);
204 OH_VideoEncoder_SetParameter(encoder_, format);
205 OH_AVFormat_Destroy(format);
206 }
207 sptr<SurfaceBuffer> buffer = frameRecord->GetSurfaceBuffer();
208 if (buffer == nullptr) {
209 MEDIA_ERR_LOG("Enqueue video buffer is empty");
210 return false;
211 }
212 std::lock_guard<std::mutex> lock(surfaceMutex_);
213 CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, false, "codecSurface_ is null");
214 SurfaceError surfaceRet = codecSurface_->AttachBufferToQueue(buffer);
215 if (surfaceRet != SURFACE_ERROR_OK) {
216 MEDIA_ERR_LOG("Failed to attach buffer, surfaceRet: %{public}d", surfaceRet);
217 // notify release buffer when attach failed
218 frameRecord->NotifyBufferRelease();
219 return false;
220 }
221 constexpr int32_t invalidFence = -1;
222 BufferFlushConfig flushConfig = {
223 .damage = {
224 .w = buffer->GetWidth(),
225 .h = buffer->GetHeight(),
226 },
227 .timestamp = frameRecord->GetTimeStamp(),
228 };
229 surfaceRet = codecSurface_->FlushBuffer(buffer, invalidFence, flushConfig);
230 CHECK_AND_RETURN_RET_LOG(surfaceRet == 0, false, "FlushBuffer failed");
231 MEDIA_DEBUG_LOG("Success frame id is : %{public}s", frameRecord->GetFrameId().c_str());
232 return true;
233 }
234
EncodeSurfaceBuffer(sptr<FrameRecord> frameRecord)235 bool VideoEncoder::EncodeSurfaceBuffer(sptr<FrameRecord> frameRecord)
236 {
237 keyFrameInterval_ = (keyFrameInterval_ == 0 ? KEY_FRAME_INTERVAL : keyFrameInterval_);
238 if (!EnqueueBuffer(frameRecord, keyFrameInterval_)) {
239 return false;
240 }
241 keyFrameInterval_--;
242 int32_t retryCount = 5;
243 while (retryCount > 0) {
244 retryCount--;
245 std::unique_lock<std::mutex> contextLock(contextMutex_);
246 CHECK_AND_RETURN_RET_LOG(context_ != nullptr, false, "VideoEncoder has been released");
247 std::unique_lock<std::mutex> lock(context_->outputMutex_);
248 bool condRet = context_->outputCond_.wait_for(lock, std::chrono::milliseconds(BUFFER_ENCODE_EXPIREATION_TIME),
249 [this]() { return !isStarted_ || !context_->outputBufferInfoQueue_.empty(); });
250 CHECK_AND_CONTINUE_LOG(!context_->outputBufferInfoQueue_.empty(),
251 "Buffer queue is empty, continue, cond ret: %{public}d", condRet);
252 sptr<CodecAVBufferInfo> bufferInfo = context_->outputBufferInfoQueue_.front();
253 MEDIA_INFO_LOG("Out buffer count: %{public}u, size: %{public}d, flag: %{public}u, pts:%{public}" PRId64 ", "
254 "timestamp:%{public}" PRId64, context_->outputFrameCount_, bufferInfo->attr.size, bufferInfo->attr.flags,
255 bufferInfo->attr.pts, frameRecord->GetTimeStamp());
256 context_->outputBufferInfoQueue_.pop();
257 context_->outputFrameCount_++;
258 lock.unlock();
259 contextLock.unlock();
260 if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_CODEC_DATA) {
261 // first return IDR frame
262 OH_AVBuffer *IDRBuffer = bufferInfo->GetCopyAVBuffer();
263 frameRecord->CacheBuffer(IDRBuffer);
264 frameRecord->SetIDRProperty(true);
265 successFrame_ = false;
266 } else if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_SYNC_FRAME) {
267 // then return I frame
268 OH_AVBuffer *tempBuffer = bufferInfo->AddCopyAVBuffer(frameRecord->encodedBuffer);
269 if (tempBuffer != nullptr) {
270 frameRecord->encodedBuffer = tempBuffer;
271 }
272 successFrame_ = true;
273 } else if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_NONE) {
274 // return P frame
275 OH_AVBuffer *PBuffer = bufferInfo->GetCopyAVBuffer();
276 frameRecord->CacheBuffer(PBuffer);
277 frameRecord->SetIDRProperty(false);
278 successFrame_ = true;
279 } else {
280 MEDIA_ERR_LOG("Flag is not acceptted number: %{public}u", bufferInfo->attr.flags);
281 int32_t ret = FreeOutputData(bufferInfo->bufferIndex);
282 CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed");
283 continue;
284 }
285 int32_t ret = FreeOutputData(bufferInfo->bufferIndex);
286 CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed");
287 if (successFrame_) {
288 MEDIA_DEBUG_LOG("Success frame id is : %{public}s, refCount: %{public}d",
289 frameRecord->GetFrameId().c_str(), frameRecord->GetSptrRefCount());
290 return true;
291 }
292 }
293 MEDIA_ERR_LOG("Failed frame id is : %{public}s", frameRecord->GetFrameId().c_str());
294 return false;
295 }
296
Release()297 int32_t VideoEncoder::Release()
298 {
299 {
300 std::lock_guard<std::mutex> lock(encoderMutex_);
301 if (encoder_ != nullptr) {
302 OH_VideoEncoder_Destroy(encoder_);
303 encoder_ = nullptr;
304 }
305 }
306 std::unique_lock<std::mutex> contextLock(contextMutex_);
307 if (context_ != nullptr) {
308 delete context_;
309 context_ = nullptr;
310 }
311 isStarted_ = false;
312 return 0;
313 }
314
SetCallback(CodecUserData * codecUserData)315 int32_t VideoEncoder::SetCallback(CodecUserData *codecUserData)
316 {
317 int32_t ret = AV_ERR_OK;
318 ret = OH_VideoEncoder_RegisterCallback(encoder_,
319 {SampleCallback::OnCodecError, SampleCallback::OnCodecFormatChange,
320 SampleCallback::OnNeedInputBuffer, SampleCallback::OnNewOutputBuffer}, codecUserData);
321 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set callback failed, ret: %{public}d", ret);
322 return 0;
323 }
324
Configure()325 int32_t VideoEncoder::Configure()
326 {
327 OH_AVFormat *format = OH_AVFormat_Create();
328 CHECK_AND_RETURN_RET_LOG(format != nullptr, 1, "AVFormat create failed");
329 int32_t bitrate = static_cast<int32_t>(pow(float(size_->width) * float(size_->height) / DEFAULT_SIZE,
330 VIDEO_BITRATE_CONSTANT) * BITRATE_22M);
331 bitrate_ = (videoCodecType_ == VideoCodecType::VIDEO_ENCODE_TYPE_AVC
332 ? static_cast<int32_t>(bitrate * HEVC_TO_AVC_FACTOR) : bitrate);
333 MEDIA_INFO_LOG("Current resolution is : %{public}d*%{public}d, encode type : %{public}d, set bitrate : %{public}d",
334 size_->width, size_->height, videoCodecType_, bitrate_);
335
336 OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, size_->width);
337 OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, size_->height);
338 OH_AVFormat_SetIntValue(format, OH_MD_KEY_ROTATION, rotation_);
339 OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, VIDEO_FRAME_RATE);
340 OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, VBR);
341 OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, bitrate_);
342 OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, VIDOE_PIXEL_FORMAT);
343 OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, INT_MAX);
344 int ret = OH_VideoEncoder_Configure(encoder_, format);
345 OH_AVFormat_Destroy(format);
346 format = nullptr;
347 CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Config failed, ret: %{public}d", ret);
348
349 return 0;
350 }
351 } // CameraStandard
352 } // OHOS