1 /*
2 * Copyright (c) 2023-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "mpeg_manager.h"
17 #include <fcntl.h>
18 #include "dp_log.h"
19 #include "dps_event_report.h"
20 #include "sync_fence.h"
21 #include "media_format.h"
22 #include "media_format_info.h"
23
24 namespace OHOS {
25 namespace CameraStandard {
26 namespace DeferredProcessing {
27 namespace {
28 const std::string VIDEO_MAKER_NAME = "DpsVideoMaker";
29 constexpr int64_t DEFAULT_TIME_TAMP = 0;
30 }
31
32 class MpegManager::VideoCodecCallback : public MediaCodecCallback {
33 public:
VideoCodecCallback(const std::weak_ptr<MpegManager> & mpegManager)34 explicit VideoCodecCallback(const std::weak_ptr<MpegManager>& mpegManager) : mpegManager_(mpegManager)
35 {
36 DP_DEBUG_LOG("entered.");
37 }
38
39 ~VideoCodecCallback() = default;
40
OnError(AVCodecErrorType errorType,int32_t errorCode)41 void OnError(AVCodecErrorType errorType, int32_t errorCode) override
42 {
43 DP_ERR_LOG("entered, errorType: %{public}d, errorCode: %{public}d", errorType, errorCode);
44 }
45
OnOutputFormatChanged(const Format & format)46 void OnOutputFormatChanged(const Format &format) override
47 {
48 DP_DEBUG_LOG("entered.");
49 }
50
OnInputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)51 void OnInputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
52 {
53 DP_DEBUG_LOG("entered, index: %{public}u", index);
54 }
55
OnOutputBufferAvailable(uint32_t index,std::shared_ptr<AVBuffer> buffer)56 void OnOutputBufferAvailable(uint32_t index, std::shared_ptr<AVBuffer> buffer) override
57 {
58 DP_CHECK_ERROR_RETURN_LOG(buffer == nullptr, "OutputBuffer is null");
59 auto manager = mpegManager_.lock();
60 DP_CHECK_ERROR_RETURN_LOG(manager == nullptr, "MpegManager is nullptr.");
61 manager->OnBufferAvailable(index, buffer);
62 }
63
64 private:
65 std::weak_ptr<MpegManager> mpegManager_;
66 };
67
68 class MpegManager::VideoMakerListener : public IBufferConsumerListener {
69 public:
VideoMakerListener(const std::weak_ptr<MpegManager> & mpegManager)70 explicit VideoMakerListener(const std::weak_ptr<MpegManager>& mpegManager) : mpegManager_(mpegManager)
71 {
72 DP_DEBUG_LOG("entered.");
73 }
74 ~VideoMakerListener() = default;
75
OnBufferAvailable()76 void OnBufferAvailable() override
77 {
78 auto manager = mpegManager_.lock();
79 DP_CHECK_ERROR_RETURN_LOG(manager == nullptr, "MpegManager is nullptr.");
80 manager->OnMakerBufferAvailable();
81 }
82
83 private:
84 std::weak_ptr<MpegManager> mpegManager_;
85 };
86
MpegManager()87 MpegManager::MpegManager()
88 {
89 DP_DEBUG_LOG("entered.");
90 mediaManager_ = std::make_unique<MediaManager>();
91 }
92
~MpegManager()93 MpegManager::~MpegManager()
94 {
95 DP_INFO_LOG("entered.");
96 remove(tempPath_.c_str());
97 if (result_ == MediaResult::PAUSE) {
98 int ret = rename(outPath_.c_str(), tempPath_.c_str());
99 if (ret != 0) {
100 DP_ERR_LOG("Rename %{public}s to %{public}s failde, ret: %{public}d",
101 outPath_.c_str(), tempPath_.c_str(), ret);
102 } else {
103 DP_INFO_LOG("Rename %{public}s to %{public}s success.", outPath_.c_str(), tempPath_.c_str());
104 }
105 }
106 if (outputFd_) {
107 fdsan_close_with_tag(outputFd_->GetFd(), LOG_DOMAIN);
108 }
109 DPSEventReport::GetInstance().ReportPartitionUsage();
110 DP_CHECK_RETURN(!tempFd_);
111 fdsan_close_with_tag(tempFd_->GetFd(), LOG_DOMAIN);
112 }
113
Init(const std::string & requestId,const sptr<IPCFileDescriptor> & inputFd)114 MediaManagerError MpegManager::Init(const std::string& requestId, const sptr<IPCFileDescriptor>& inputFd)
115 {
116 DP_DEBUG_LOG("entered.");
117 outputFd_ = GetFileFd(requestId, O_CREAT | O_RDWR, OUT_TAG);
118 DP_CHECK_ERROR_RETURN_RET_LOG(outputFd_ == nullptr, ERROR_FAIL, "Output video create failde.");
119
120 tempFd_ = GetFileFd(requestId, O_RDONLY, TEMP_TAG);
121 DP_CHECK_ERROR_RETURN_RET_LOG(tempFd_ == nullptr, ERROR_FAIL, "Temp video create failde.");
122
123 auto ret = mediaManager_->Create(inputFd->GetFd(), outputFd_->GetFd(), tempFd_->GetFd());
124 DP_CHECK_ERROR_RETURN_RET_LOG(ret != OK, ERROR_FAIL, "Media manager create failde.");
125
126 {
127 std::lock_guard<std::mutex> lock(mediaInfoMutex_);
128 mediaManager_->GetMediaInfo(mediaInfo_);
129 }
130 DP_CHECK_ERROR_RETURN_RET_LOG(InitVideoCodec() != OK, ERROR_FAIL, "Init video codec failde.");
131
132 DP_CHECK_ERROR_RETURN_RET_LOG(InitVideoMakerSurface() != OK, ERROR_FAIL, "Init video maker surface failde.");
133 isRunning_.store(true);
134 return OK;
135 }
136
UnInit(const MediaResult result)137 MediaManagerError MpegManager::UnInit(const MediaResult result)
138 {
139 DP_DEBUG_LOG("entered.");
140 DP_CHECK_RETURN_RET(!isRunning_.load(), OK);
141 result_ = result;
142 if (result == MediaResult::PAUSE) {
143 if (mediaManager_->Pause() == PAUSE_ABNORMAL) {
144 remove(outPath_.c_str());
145 }
146 } else {
147 mediaManager_->Stop();
148 }
149 UnInitVideoCodec();
150 isRunning_.store(false);
151 return OK;
152 }
153
GetSurface()154 sptr<Surface> MpegManager::GetSurface()
155 {
156 return codecSurface_;
157 }
158
GetMakerSurface()159 sptr<Surface> MpegManager::GetMakerSurface()
160 {
161 return makerSurface_;
162 }
163
GetProcessTimeStamp()164 uint64_t MpegManager::GetProcessTimeStamp()
165 {
166 std::lock_guard<std::mutex> lock(mediaInfoMutex_);
167 DP_CHECK_RETURN_RET_LOG(mediaInfo_ == nullptr, DEFAULT_TIME_TAMP, "mediaInfo_ is nullptr");
168 DP_CHECK_RETURN_RET(mediaInfo_->recoverTime < DEFAULT_TIME_TAMP, DEFAULT_TIME_TAMP);
169 return static_cast<uint64_t>(mediaInfo_->recoverTime);
170 }
171
NotifyEnd()172 MediaManagerError MpegManager::NotifyEnd()
173 {
174 auto ret = encoder_->NotifyEos();
175 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL, "Video codec notify end failde.");
176 return OK;
177 }
178
GetResultFd()179 sptr<IPCFileDescriptor> MpegManager::GetResultFd()
180 {
181 return outputFd_;
182 }
183
AddUserMeta(std::unique_ptr<MediaUserInfo> userInfo)184 void MpegManager::AddUserMeta(std::unique_ptr<MediaUserInfo> userInfo)
185 {
186 DP_INFO_LOG("SetUserMeta entered.");
187 std::shared_ptr<Meta> userMeta = std::make_shared<Meta>();
188 if (userInfo->scalingFactor != DEFAULT_SCALING_FACTOR) {
189 userMeta->SetData(SCALING_FACTOR_KEY, userInfo->scalingFactor);
190 }
191 if (!userInfo->interpolationFramePts.empty()) {
192 userMeta->SetData(INTERPOLATION_FRAME_PTS_KEY, userInfo->interpolationFramePts);
193 }
194 if (!userInfo->stageVid.empty()) {
195 userMeta->SetData(STAGE_VID_KEY, userInfo->stageVid);
196 }
197 mediaManager_->AddUserMeta(userMeta);
198 }
199
SetMarkSize(int32_t size)200 void MpegManager::SetMarkSize(int32_t size)
201 {
202 DP_CHECK_EXECUTE(mediaManager_, mediaManager_->SetMarkSize(size));
203 }
204
InitVideoCodec()205 MediaManagerError MpegManager::InitVideoCodec()
206 {
207 DP_INFO_LOG("DPS_VIDEO: Create video codec.");
208 CodecInfo codecInfo;
209 {
210 std::lock_guard<std::mutex> lock(mediaInfoMutex_);
211 DP_CHECK_RETURN_RET_LOG(mediaInfo_ == nullptr, OK, "mediaInfo_ is nullptr");
212 codecInfo = mediaInfo_->codecInfo;
213 }
214 encoder_ = VideoEncoderFactory::CreateByMime(codecInfo.mimeType);
215 DP_CHECK_ERROR_RETURN_RET_LOG(encoder_ == nullptr, ERROR_FAIL, "Video codec create failde.");
216
217 auto callback = std::make_shared<VideoCodecCallback>(weak_from_this());
218 auto ret = encoder_->SetCallback(callback);
219 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL,
220 "Video codec set callback failde, ret:%{public}d.", ret);
221
222 Format videoFormat;
223 if (codecInfo.mimeType == MIME_VIDEO_HEVC) {
224 videoFormat.PutIntValue(Tag::VIDEO_COLOR_RANGE, static_cast<int32_t>(codecInfo.colorRange));
225 videoFormat.PutIntValue(Tag::VIDEO_COLOR_PRIMARIES, static_cast<int32_t>(codecInfo.colorPrimary));
226 videoFormat.PutIntValue(Tag::VIDEO_COLOR_TRC, static_cast<int32_t>(codecInfo.colorTransferCharacter));
227 videoFormat.PutIntValue(Tag::MEDIA_LEVEL, codecInfo.level);
228 videoFormat.PutIntValue(Tag::MEDIA_PROFILE, codecInfo.profile);
229 }
230 videoFormat.PutIntValue(Tag::VIDEO_PIXEL_FORMAT, static_cast<int32_t>(Media::Plugins::VideoPixelFormat::NV12));
231 videoFormat.PutIntValue(Tag::VIDEO_ENCODE_BITRATE_MODE, codecInfo.bitMode);
232 videoFormat.PutStringValue(Tag::MIME_TYPE, codecInfo.mimeType);
233 videoFormat.PutLongValue(Tag::MEDIA_BITRATE, codecInfo.bitRate);
234 videoFormat.PutDoubleValue(Tag::VIDEO_FRAME_RATE, codecInfo.fps);
235 videoFormat.PutIntValue(Tag::VIDEO_WIDTH, codecInfo.width);
236 videoFormat.PutIntValue(Tag::VIDEO_HEIGHT, codecInfo.height);
237 videoFormat.PutIntValue(Tag::VIDEO_ROTATION, codecInfo.rotation);
238 videoFormat.PutLongValue(Tag::MEDIA_DURATION, codecInfo.duration);
239
240 ret = encoder_->Configure(videoFormat);
241 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL,
242 "Video codec configure failde, ret: %{public}d.", ret);
243
244 codecSurface_ = encoder_->CreateInputSurface();
245 ret = encoder_->Prepare();
246 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL, "Video codec prepare failde.");
247
248 ret = encoder_->Start();
249 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL, "Video codec start failde.");
250
251 return OK;
252 }
253
UnInitVideoCodec()254 void MpegManager::UnInitVideoCodec()
255 {
256 DP_DEBUG_LOG("entered.");
257 if (isRunning_) {
258 encoder_->Stop();
259 }
260 DP_CHECK_RETURN(encoder_ == nullptr);
261 encoder_->Release();
262 }
263
InitVideoMakerSurface()264 MediaManagerError MpegManager::InitVideoMakerSurface()
265 {
266 DP_INFO_LOG("DPS_VIDEO: Create video maker surface.");
267 makerSurface_ = Surface::CreateSurfaceAsConsumer(VIDEO_MAKER_NAME);
268 DP_CHECK_ERROR_RETURN_RET_LOG(makerSurface_ == nullptr, ERROR_FAIL, "Video maker surface create failde.");
269
270 auto listener = sptr<VideoMakerListener>::MakeSptr(weak_from_this());
271 auto ret = makerSurface_->RegisterConsumerListener((sptr<IBufferConsumerListener> &)listener);
272 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL,
273 "Video maker surface set callback failde, ret:%{public}d.", ret);
274
275 return OK;
276 }
277
UnInitVideoMaker()278 void MpegManager::UnInitVideoMaker()
279 {
280 DP_DEBUG_LOG("entered.");
281 }
282
OnBufferAvailable(uint32_t index,const std::shared_ptr<AVBuffer> & buffer)283 void MpegManager::OnBufferAvailable(uint32_t index, const std::shared_ptr<AVBuffer>& buffer)
284 {
285 DP_DEBUG_LOG("OnBufferAvailable: pts: %{public}" PRId64 ", duration: %{public}" PRId64,
286 buffer->pts_, buffer->duration_);
287 auto ret = mediaManager_->WriteSample(Media::Plugins::MediaType::VIDEO, buffer);
288 DP_CHECK_ERROR_PRINT_LOG(ret != OK, "Video codec write failde.");
289
290 ret = ReleaseBuffer(index);
291 DP_CHECK_ERROR_PRINT_LOG(ret != OK, "Video codec release buffer failde.");
292 }
293
ReleaseBuffer(uint32_t index)294 MediaManagerError MpegManager::ReleaseBuffer(uint32_t index)
295 {
296 auto ret = encoder_->ReleaseOutputBuffer(index);
297 DP_CHECK_ERROR_RETURN_RET_LOG(ret != static_cast<int32_t>(OK), ERROR_FAIL, "Video codec release buffer failde.");
298 return OK;
299 }
300
OnMakerBufferAvailable()301 void MpegManager::OnMakerBufferAvailable()
302 {
303 DP_CHECK_ERROR_RETURN_LOG(makerSurface_ == nullptr, "MakerSurface is nullptr.");
304 DP_DEBUG_LOG("OnMakerBufferAvailable: surface size: %{public}u", makerSurface_->GetQueueSize());
305
306 int64_t timestamp;
307 auto buffer = AcquireMakerBuffer(timestamp);
308 DP_CHECK_ERROR_RETURN_LOG(buffer == nullptr, "MakerBuffer is nullptr.");
309 if (timestamp < 0) {
310 ReleaseMakerBuffer(buffer);
311 return;
312 }
313
314 std::lock_guard<std::mutex> lock(makerMutex_);
315 auto makerBuffer = AVBuffer::CreateAVBuffer(buffer);
316 DP_CHECK_ERROR_RETURN_LOG(makerBuffer == nullptr, "CreateAVBuffer failde.");
317
318 makerBuffer->pts_ = timestamp;
319 makerBuffer->memory_->SetSize(buffer->GetWidth());
320 DP_DEBUG_LOG("MakerBuffer pts %{public}" PRId64 "marke size: %{public}d",
321 makerBuffer->pts_, makerBuffer->memory_->GetSize());
322 auto ret = mediaManager_->WriteSample(Media::Plugins::MediaType::TIMEDMETA, makerBuffer);
323 DP_CHECK_ERROR_PRINT_LOG(ret != OK, "Video maker data write failde.");
324
325 ret = ReleaseMakerBuffer(buffer);
326 DP_CHECK_ERROR_PRINT_LOG(ret != OK, "Video maker data release buffer failde.");
327 }
328
AcquireMakerBuffer(int64_t & timestamp)329 sptr<SurfaceBuffer> MpegManager::AcquireMakerBuffer(int64_t& timestamp)
330 {
331 OHOS::Rect damage;
332 sptr<SurfaceBuffer> buffer;
333 sptr<SyncFence> syncFence = SyncFence::INVALID_FENCE;
334 auto surfaceRet = makerSurface_->AcquireBuffer(buffer, syncFence, timestamp, damage);
335 DP_CHECK_ERROR_RETURN_RET_LOG(surfaceRet != SURFACE_ERROR_OK, nullptr,
336 "Acquire maker buffer failed, ret: %{public}d", surfaceRet);
337
338 surfaceRet = makerSurface_->DetachBufferFromQueue(buffer);
339 DP_CHECK_ERROR_RETURN_RET_LOG(surfaceRet != SURFACE_ERROR_OK, nullptr,
340 "Detach maker buffer failed, ret: %{public}d", surfaceRet);
341 return buffer;
342 }
343
ReleaseMakerBuffer(sptr<SurfaceBuffer> & buffer)344 MediaManagerError MpegManager::ReleaseMakerBuffer(sptr<SurfaceBuffer>& buffer)
345 {
346 auto ret = makerSurface_->AttachBufferToQueue(buffer);
347 DP_CHECK_ERROR_RETURN_RET_LOG(ret != SURFACE_ERROR_OK, ERROR_FAIL, "Attach maker buffer failde.");
348
349 ret = makerSurface_->ReleaseBuffer(buffer, -1);
350 DP_CHECK_ERROR_RETURN_RET_LOG(ret != SURFACE_ERROR_OK, ERROR_FAIL, "Release maker buffer failde.");
351 return OK;
352 }
353
GetFileFd(const std::string & requestId,int flags,const std::string & tag)354 sptr<IPCFileDescriptor> MpegManager::GetFileFd(const std::string& requestId, int flags, const std::string& tag)
355 {
356 std::string path = PATH + requestId + tag;
357 if (tag == TEMP_TAG) {
358 tempPath_ = path;
359 } else {
360 outPath_ = path;
361 }
362 int fd = open(path.c_str(), flags, S_IRUSR | S_IWUSR);
363 fdsan_exchange_owner_tag(fd, 0, LOG_DOMAIN);
364 DP_DEBUG_LOG("GetFileFd path: %{public}s, fd: %{public}d", path.c_str(), fd);
365 return sptr<IPCFileDescriptor>::MakeSptr(fd);
366 }
367 } // namespace DeferredProcessing
368 } // namespace CameraStandard
369 } // namespace OHOS