1 /*
2 * Copyright (c) 2022-2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "image_source_encoder.h"
17
18 #include <dlfcn.h>
19 #include <fstream>
20 #include <iostream>
21 #include <securec.h>
22 #include <csetjmp>
23 #include <stddef.h>
24 #include <string>
25 #include <ctime>
26
27 #include "dscreen_constants.h"
28 #include "dscreen_errcode.h"
29 #include "dscreen_hisysevent.h"
30 #include "dscreen_log.h"
31 #include "jpeglib.h"
32 #ifdef __LP64__
33 static const std::string LIB_LOAD_PATH = "/system/lib64/libdistributed_screen_dbg_itf.z.so";
34 #else
35 static const std::string LIB_LOAD_PATH = "/system/lib/libdistributed_screen_dbg_itf.z.so";
36 #endif
37 using GetDscreenDBGItfFunc = OHOS::DistributedHardware::IDScreenDBGItf* (*)();
38 using GetImageDirtyFunc = OHOS::DistributedHardware::IImageSetDirty* (*)();
39 namespace OHOS {
40 namespace DistributedHardware {
OnBufferAvailable()41 void ConsumerBufferListener::OnBufferAvailable()
42 {
43 DHLOGI("%s: OnBufferAvailable, receiv data from RS.", LOG_TAG);
44 encoder_->ConsumeSurface();
45 }
46
InitDscreenDBG()47 void ImageSourceEncoder::InitDscreenDBG()
48 {
49 char path[PATH_MAX + 1] = {0x00};
50 if (LIB_LOAD_PATH.length() > PATH_MAX || realpath(LIB_LOAD_PATH.c_str(), path) == nullptr) {
51 DHLOGE("File connicailization failed.");
52 return;
53 }
54 pHandler_ = dlopen(path, RTLD_LAZY | RTLD_NODELETE | RTLD_GLOBAL);
55 if (pHandler_ == nullptr) {
56 DHLOGE("%s: handler load failed, fail reason: %s.", path, dlerror());
57 return;
58 }
59 GetDscreenDBGItfFunc getDscreenDBGItfFunc = (GetDscreenDBGItfFunc)dlsym(pHandler_, GET_DBG_ITF_FUNC.c_str());
60 GetImageDirtyFunc getImageDirtyFunc = (GetImageDirtyFunc)dlsym(pHandler_, GET_IMAGE_DIRTY_FUNC.c_str());
61 if (getDscreenDBGItfFunc == nullptr || getImageDirtyFunc ==nullptr) {
62 DHLOGE("get FUNC failed, failed reason: %s.", dlerror());
63 return;
64 }
65 dscreenDbgItfPtr_ = getDscreenDBGItfFunc();
66 imageSetDirtyPtr_ = getImageDirtyFunc();
67 if (dscreenDbgItfPtr_ == nullptr || imageSetDirtyPtr_ == nullptr) {
68 DHLOGE("Get interface failed.");
69 return;
70 }
71 DHLOGI("InitDscreenDBG success.");
72 dscreenDbgItfPtr_->Init();
73 }
74
ConsumeSurface()75 void ImageSourceEncoder::ConsumeSurface()
76 {
77 DHLOGI("%s: ConsumeSurface.", LOG_TAG);
78 std::unique_lock<std::mutex> bufLock(bufferMtx_);
79 if (consumerSurface_ == nullptr) {
80 DHLOGE("%s: consumerSurface_ is nullptr.", LOG_TAG);
81 return;
82 }
83 sptr<SurfaceBuffer> surfaceBuffer = nullptr;
84 syncFence_ = SyncFence::INVALID_FENCE;
85 int64_t timestamp = 0;
86 OHOS::Rect damage = {0, 0, 0, 0};
87 SurfaceError surfaceErr = consumerSurface_->AcquireBuffer(surfaceBuffer, syncFence_, timestamp, damage);
88 if (surfaceErr != SURFACE_ERROR_OK) {
89 DHLOGE("%s: consumerSurface_ acquire buffer failed, errcode: %" PRId32, LOG_TAG, surfaceErr);
90 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
91 return;
92 }
93 int32_t retcode = syncFence_->Wait(SURFACE_SYNC_FENCE_TIMEOUT);
94 if (retcode == -ETIME) {
95 DHLOGE("%s: Sync fence wait timeout, retcode is %." PRId32, LOG_TAG, retcode);
96 return;
97 }
98 if (pHandler_ != nullptr) {
99 eventContent_.clear();
100 eventContent_ = imageSetDirtyPtr_->GetDamage();
101 }
102 std::vector<OHOS::Rect> damages = VecToDamage(eventContent_);
103 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
104 if (listener == nullptr) {
105 DHLOGE("%s: Processor listener is null", LOG_TAG);
106 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
107 return;
108 }
109 listener->OnDamageProcessDone(surfaceBuffer, damages);
110 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
111 }
112
ConfigureEncoder(const VideoParam & configParam)113 int32_t ImageSourceEncoder::ConfigureEncoder(const VideoParam &configParam)
114 {
115 DHLOGI("%s: ConfigureEncoder.", LOG_TAG);
116 int32_t ret = InitVideoEncoder(configParam);
117 if (ret != DH_SUCCESS) {
118 DHLOGE("%s: Init encoder failed ret: %" PRId32, LOG_TAG, ret);
119 return ret;
120 }
121
122 ret = SetEncoderFormat(configParam);
123 if (ret != DH_SUCCESS) {
124 DHLOGE("%s: Set encoder format failed ret: %" PRId32, LOG_TAG, ret);
125 return ret;
126 }
127
128 encoderSurface_ = videoEncoder_->CreateInputSurface();
129 if (encoderSurface_ == nullptr) {
130 DHLOGE("%s: Create encoder surface failed.", LOG_TAG);
131 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
132 }
133 configParam_ = configParam;
134 ret = AddSurface();
135 if (ret != DH_SUCCESS) {
136 DHLOGE("%s: Add surface failed ret: %." PRId32, LOG_TAG, ret);
137 consumerSurface_ = nullptr;
138 producerSurface_ = nullptr;
139 return ret;
140 }
141 return DH_SUCCESS;
142 }
143
AddSurface()144 int32_t ImageSourceEncoder::AddSurface()
145 {
146 DHLOGI("%s: AddSurface.", LOG_TAG);
147 consumerSurface_ = IConsumerSurface::Create();
148 if (consumerSurface_ == nullptr) {
149 DHLOGE("%s: creat consumer surface failed.", LOG_TAG);
150 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
151 }
152 sptr<IBufferProducer> producer = consumerSurface_->GetProducer();
153 if (producer == nullptr) {
154 DHLOGE("%s: Creat producer surface failed.", LOG_TAG);
155 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
156 }
157 producerSurface_ = Surface::CreateSurfaceAsProducer(producer);
158 if (producerSurface_ == nullptr) {
159 DHLOGE("%s: Create preducer surface failed.", LOG_TAG);
160 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
161 }
162 consumerBufferListener_ = new ConsumerBufferListener(shared_from_this());
163 consumerSurface_->RegisterConsumerListener(consumerBufferListener_);
164 return DH_SUCCESS;
165 }
166
GetConsumerSurface()167 sptr<Surface> ImageSourceEncoder::GetConsumerSurface()
168 {
169 DHLOGI("%s: GetConsumerSurface.", LOG_TAG);
170 return consumerSurface_;
171 }
172
GetInputSurface()173 sptr<Surface> &ImageSourceEncoder::GetInputSurface()
174 {
175 DHLOGI("%s: GetInputSurface.", LOG_TAG);
176 if (producerSurface_ == nullptr || !configParam_.GetPartialRefreshFlag()) {
177 return encoderSurface_;
178 }
179 return producerSurface_;
180 }
181
VecToDamage(std::vector<std::vector<int32_t>> eventContent)182 std::vector<OHOS::Rect> ImageSourceEncoder::VecToDamage(std::vector<std::vector<int32_t>> eventContent)
183 {
184 DHLOGI("%s: VecToDamage.", LOG_TAG);
185 std::vector<OHOS::Rect> damages;
186 for (auto item : eventContent) {
187 OHOS::Rect damage = {0, 0, 0, 0};
188 damage.x = item[0];
189 damage.y = item[1];
190 damage.w = item[2];
191 damage.h = item[3];
192 damages.push_back(damage);
193 }
194 return damages;
195 }
196
GetEncoderInputSurfaceBuffer()197 sptr<SurfaceBuffer> ImageSourceEncoder::GetEncoderInputSurfaceBuffer()
198 {
199 DHLOGI("%s: GetEncoderInputSurfaceBuffer.", LOG_TAG);
200 OHOS::BufferRequestConfig requestConfig;
201 requestConfig.width = static_cast<int32_t>(configParam_.GetVideoWidth());
202 requestConfig.height = static_cast<int32_t>(configParam_.GetVideoHeight());
203 requestConfig.usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA;
204 requestConfig.strideAlignment = STRIDE_ALIGNMENT;
205 requestConfig.format = GraphicPixelFormat::GRAPHIC_PIXEL_FMT_RGBA_8888;
206 requestConfig.timeout = 0;
207 sptr<OHOS::SurfaceBuffer> encoderSurfaceBuffer = nullptr;
208 int32_t releaseFence = -1;
209 SurfaceError surfaceErr = encoderSurface_->RequestBuffer(encoderSurfaceBuffer, releaseFence, requestConfig);
210 if (surfaceErr != GSERROR_OK || encoderSurfaceBuffer == nullptr) {
211 DHLOGE("%s: RequestBuffer failed.", LOG_TAG);
212 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
213 }
214 return encoderSurfaceBuffer;
215 }
216
FeedEncoderData(sptr<SurfaceBuffer> & surfaceBuffer)217 int32_t ImageSourceEncoder::FeedEncoderData(sptr<SurfaceBuffer> &surfaceBuffer)
218 {
219 sptr<OHOS::SurfaceBuffer> encoderSurfaceBuffer = GetEncoderInputSurfaceBuffer();
220 if (encoderSurfaceBuffer == nullptr) {
221 DHLOGE("Get encoder input producer surface buffer failed.");
222 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
223 }
224 uint32_t screenDataSize = configParam_.GetVideoWidth() * configParam_.GetVideoHeight() * RGBA_CHROMA;
225 auto encoderSurfaceAddr = static_cast<uint8_t*>(encoderSurfaceBuffer->GetVirAddr());
226 auto surfaceAddr = static_cast<uint8_t*>(surfaceBuffer->GetVirAddr());
227 int32_t ret = memcpy_s(encoderSurfaceAddr, screenDataSize, surfaceAddr, screenDataSize);
228 if (ret != EOK) {
229 DHLOGE("%s: surfaceBuffer memcpy_s run failed.", LOG_TAG);
230 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
231 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
232 return ret;
233 }
234 BufferFlushConfig flushConfig = { {0, 0, encoderSurfaceBuffer->GetWidth(), encoderSurfaceBuffer->GetHeight()}, 0};
235 DHLOGI("%s: FeedEncoderData to H264 encoder.", LOG_TAG);
236 SurfaceError surfaceErr = encoderSurface_->FlushBuffer(encoderSurfaceBuffer, -1, flushConfig);
237 if (surfaceErr != SURFACE_ERROR_OK) {
238 DHLOGE("%s: encoderSurface_ flush buffer failed.", LOG_TAG);
239 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
240 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
241 return surfaceErr;
242 }
243 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
244 return DH_SUCCESS;
245 }
246
ReleaseEncoder()247 int32_t ImageSourceEncoder::ReleaseEncoder()
248 {
249 DHLOGI("%s: ReleaseEncoder.", LOG_TAG);
250 if (videoEncoder_ == nullptr) {
251 DHLOGE("%s: Encoder is null.", LOG_TAG);
252 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
253 }
254
255 int32_t ret = videoEncoder_->Release();
256 if (ret != Media::MSERR_OK) {
257 DHLOGE("%s: Release encoder failed.", LOG_TAG);
258 return ERR_DH_SCREEN_CODEC_RELEASE_FAILED;
259 }
260 if (pHandler_ != nullptr) {
261 dscreenDbgItfPtr_->Release();
262 dlclose(pHandler_);
263 }
264 encodeVideoCallback_ = nullptr;
265 videoEncoder_ = nullptr;
266 consumerBufferListener_ = nullptr;
267 return DH_SUCCESS;
268 }
269
StartEncoder()270 int32_t ImageSourceEncoder::StartEncoder()
271 {
272 DHLOGI("%s: StartEncoder.", LOG_TAG);
273 if (videoEncoder_ == nullptr) {
274 DHLOGE("%s: Encoder is null.", LOG_TAG);
275 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
276 }
277
278 int32_t ret = videoEncoder_->Prepare();
279 if (ret != Media::MSERR_OK) {
280 DHLOGE("%s: Prepare encoder failed.", LOG_TAG);
281 return ERR_DH_SCREEN_CODEC_PREPARE_FAILED;
282 }
283
284 ret = videoEncoder_->Start();
285 if (ret != Media::MSERR_OK) {
286 DHLOGE("%s: Start encoder failed.", LOG_TAG);
287 return ERR_DH_SCREEN_CODEC_START_FAILED;
288 }
289 InitDscreenDBG();
290 return DH_SUCCESS;
291 }
292
StopEncoder()293 int32_t ImageSourceEncoder::StopEncoder()
294 {
295 DHLOGI("%s: StopEncoder.", LOG_TAG);
296 if (videoEncoder_ == nullptr) {
297 DHLOGE("%s: Encoder is null.", LOG_TAG);
298 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
299 }
300
301 int32_t ret = videoEncoder_->Flush();
302 if (ret != Media::MSERR_OK) {
303 DHLOGE("%s: Flush encoder failed.", LOG_TAG);
304 }
305
306 ret = videoEncoder_->Stop();
307 if (ret != Media::MSERR_OK) {
308 DHLOGE("%s: Stop encoder failed.", LOG_TAG);
309 return ERR_DH_SCREEN_CODEC_STOP_FAILED;
310 }
311 ret = consumerSurface_->UnregisterConsumerListener();
312 if (ret != SURFACE_ERROR_OK) {
313 DHLOGE("Unregister Consumer Listener failed.");
314 }
315 consumerSurface_ = nullptr;
316 producerSurface_ = nullptr;
317 return DH_SUCCESS;
318 }
319
InitVideoEncoder(const VideoParam & configParam)320 int32_t ImageSourceEncoder::InitVideoEncoder(const VideoParam &configParam)
321 {
322 DHLOGI("%s: InitVideoEncoder.", LOG_TAG);
323 switch (configParam.GetCodecType()) {
324 case VIDEO_CODEC_TYPE_VIDEO_H264:
325 videoEncoder_ = Media::VideoEncoderFactory::CreateByMime("video/avc");
326 break;
327 case VIDEO_CODEC_TYPE_VIDEO_H265:
328 videoEncoder_ = Media::VideoEncoderFactory::CreateByMime("video/hevc");
329 break;
330 case VIDEO_CODEC_TYPE_VIDEO_MPEG4:
331 videoEncoder_ = Media::VideoEncoderFactory::CreateByMime("video/mp4v-es");
332 break;
333 default:
334 DHLOGE("%s: codecType is invalid!", LOG_TAG);
335 videoEncoder_ = nullptr;
336 }
337
338 if (videoEncoder_ == nullptr) {
339 DHLOGE("%s: Create videoEncoder failed.", LOG_TAG);
340 return ERR_DH_SCREEN_TRANS_CREATE_CODEC_FAILED;
341 }
342
343 encodeVideoCallback_ = std::make_shared<ImageEncoderCallback>(shared_from_this());
344 int32_t ret = videoEncoder_->SetCallback(encodeVideoCallback_);
345 if (ret != Media::MSERR_OK) {
346 DHLOGE("%s: Set codec callback failed.", LOG_TAG);
347 return ERR_DH_SCREEN_CODEC_SET_CALLBACK_FAILED;
348 }
349
350 return DH_SUCCESS;
351 }
352
SetEncoderFormat(const VideoParam & configParam)353 int32_t ImageSourceEncoder::SetEncoderFormat(const VideoParam &configParam)
354 {
355 DHLOGI("%s: SetEncoderFormat.", LOG_TAG);
356 if (videoEncoder_ == nullptr) {
357 DHLOGE("%s: Encoder is null.", LOG_TAG);
358 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
359 }
360
361 switch (configParam.GetCodecType()) {
362 case VIDEO_CODEC_TYPE_VIDEO_H264:
363 imageFormat_.PutStringValue("codec_mime", "video/avc");
364 break;
365 case VIDEO_CODEC_TYPE_VIDEO_H265:
366 imageFormat_.PutStringValue("codec_mime", "video/hevc");
367 break;
368 case VIDEO_CODEC_TYPE_VIDEO_MPEG4:
369 imageFormat_.PutStringValue("codec_mime", "video/mp4v-es");
370 break;
371 default:
372 DHLOGE("%s: Codec type is invalid.", LOG_TAG);
373 return ERR_DH_SCREEN_TRANS_ILLEGAL_PARAM;
374 }
375 switch (configParam.GetVideoFormat()) {
376 case VIDEO_DATA_FORMAT_YUVI420:
377 imageFormat_.PutIntValue("pixel_format", Media::VideoPixelFormat::YUVI420);
378 break;
379 case VIDEO_DATA_FORMAT_NV12:
380 imageFormat_.PutIntValue("pixel_format", Media::VideoPixelFormat::NV12);
381 break;
382 case VIDEO_DATA_FORMAT_NV21:
383 imageFormat_.PutIntValue("pixel_format", Media::VideoPixelFormat::NV21);
384 break;
385 case VIDEO_DATA_FORMAT_RGBA8888:
386 imageFormat_.PutIntValue("pixel_format", Media::VideoPixelFormat::RGBA);
387 break;
388 default:
389 DHLOGE("%s: Video format is invalid.", LOG_TAG);
390 return ERR_DH_SCREEN_TRANS_ILLEGAL_PARAM;
391 }
392 imageFormat_.PutLongValue("max_input_size", MAX_YUV420_BUFFER_SIZE);
393 imageFormat_.PutIntValue("width", configParam.GetVideoWidth());
394 imageFormat_.PutIntValue("height", configParam.GetVideoHeight());
395 imageFormat_.PutIntValue("frame_rate", configParam.GetFps());
396
397 int32_t ret = videoEncoder_->Configure(imageFormat_);
398 if (ret != Media::MSERR_OK) {
399 DHLOGE("%s: Configure encoder failed.", LOG_TAG);
400 return ERR_DH_SCREEN_CODEC_CONFIGURE_FAILED;
401 }
402 return DH_SUCCESS;
403 }
404
OnError(Media::AVCodecErrorType errorType,int32_t errorCode)405 void ImageSourceEncoder::OnError(Media::AVCodecErrorType errorType, int32_t errorCode)
406 {
407 DHLOGI("%s: Encoder error, errorType:%" PRId32", errorCode:%" PRId32, LOG_TAG, errorType, errorCode);
408 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
409 if (listener == nullptr) {
410 DHLOGE("%s: Processor listener is null", LOG_TAG);
411 return;
412 }
413 listener->OnProcessorStateNotify(errorCode);
414 }
415
OnOutputBufferAvailable(uint32_t index,Media::AVCodecBufferInfo info,Media::AVCodecBufferFlag flag)416 void ImageSourceEncoder::OnOutputBufferAvailable(uint32_t index, Media::AVCodecBufferInfo info,
417 Media::AVCodecBufferFlag flag)
418 {
419 DHLOGD("%s: OnOutputBufferAvailable, receiv H264 data from encoder.", LOG_TAG);
420 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
421 if (listener == nullptr) {
422 DHLOGE("%s: Processor listener is null", LOG_TAG);
423 return;
424 }
425 if (videoEncoder_ == nullptr) {
426 DHLOGE("%s: Encoder is null.", LOG_TAG);
427 return;
428 }
429
430 encoderBufferInfo_ = info;
431 videoSharedMemory_ = videoEncoder_->GetOutputBuffer(index);
432 if (videoSharedMemory_ == nullptr) {
433 DHLOGE("%s: GetOutputBuffer failed.", LOG_TAG);
434 return;
435 }
436
437 size_t dataSize = static_cast<size_t>(info.size);
438 if (dataSize == 0 || dataSize > DATA_BUFFER_MAX_SIZE) {
439 DHLOGE("%s:OnOutputBufferAvailable params invalid, size: %" PRId32, LOG_TAG, dataSize);
440 return;
441 }
442 auto dataBuf = std::make_shared<DataBuffer>(dataSize);
443 int32_t ret = memcpy_s(dataBuf->Data(), dataBuf->Capacity(), videoSharedMemory_->GetBase(), dataSize);
444 if (ret != EOK) {
445 DHLOGE("%s: Copy data failed.", LOG_TAG);
446 return;
447 }
448 dataBuf->SetDataType(VIDEO_FULL_SCREEN_DATA);
449 dataBuf->SetSize(dataSize);
450 listener->OnImageProcessDone(dataBuf);
451 ret = videoEncoder_->ReleaseOutputBuffer(index);
452 if (ret != Media::MSERR_OK) {
453 DHLOGE("%s: videoEncoder ReleaseOutputBuffer failed.", LOG_TAG);
454 }
455 }
456
OnInputBufferAvailable(uint32_t index)457 void ImageSourceEncoder::OnInputBufferAvailable(uint32_t index)
458 {
459 (void) index;
460 }
461
OnOutputFormatChanged(const Media::Format & format)462 void ImageSourceEncoder::OnOutputFormatChanged(const Media::Format &format)
463 {
464 (void) format;
465 }
466 } // namespace DistributedHardware
467 } // namespace OHOS