1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "image_source_encoder.h"
17
18 #include <dlfcn.h>
19 #include <fstream>
20 #include <iostream>
21 #include <securec.h>
22 #include <csetjmp>
23 #include <cstddef>
24 #include <string>
25 #include <ctime>
26
27 #include "dscreen_constants.h"
28 #include "dscreen_errcode.h"
29 #include "dscreen_hisysevent.h"
30 #include "dscreen_log.h"
31 #include "jpeglib.h"
32
33 using GetDscreenDBGItfFunc = OHOS::DistributedHardware::IDScreenDBGItf* (*)();
34 using GetImageDirtyFunc = OHOS::DistributedHardware::IImageSetDirty* (*)();
35 namespace OHOS {
36 namespace DistributedHardware {
37 const std::string DBG_SO_NAME = "libdistributed_screen_dbg_itf.z.so";
OnBufferAvailable()38 void ConsumerBufferListener::OnBufferAvailable()
39 {
40 DHLOGI("%{public}s: OnBufferAvailable, receiv data from RS.", DSCREEN_LOG_TAG);
41 encoder_->ConsumeSurface();
42 }
43
InitDscreenDBG()44 void ImageSourceEncoder::InitDscreenDBG()
45 {
46 pHandler_ = dlopen(DBG_SO_NAME.c_str(), RTLD_LAZY | RTLD_NODELETE | RTLD_GLOBAL);
47 if (pHandler_ == nullptr) {
48 DHLOGE("so: %{public}s load failed, fail reason: %{public}s.", DBG_SO_NAME.c_str(), dlerror());
49 return;
50 }
51 GetDscreenDBGItfFunc getDscreenDBGItfFunc = (GetDscreenDBGItfFunc)dlsym(pHandler_, GET_DBG_ITF_FUNC.c_str());
52 GetImageDirtyFunc getImageDirtyFunc = (GetImageDirtyFunc)dlsym(pHandler_, GET_IMAGE_DIRTY_FUNC.c_str());
53 if (getDscreenDBGItfFunc == nullptr || getImageDirtyFunc == nullptr) {
54 DHLOGE("get FUNC failed, failed reason: %{public}s.", dlerror());
55 return;
56 }
57 dscreenDbgItfPtr_ = getDscreenDBGItfFunc();
58 imageSetDirtyPtr_ = getImageDirtyFunc();
59 if (dscreenDbgItfPtr_ == nullptr || imageSetDirtyPtr_ == nullptr) {
60 DHLOGE("Get interface failed.");
61 return;
62 }
63 DHLOGI("InitDscreenDBG success.");
64 dscreenDbgItfPtr_->Init();
65 }
66
ConsumeSurface()67 void ImageSourceEncoder::ConsumeSurface()
68 {
69 DHLOGI("%{public}s: ConsumeSurface.", DSCREEN_LOG_TAG);
70 std::unique_lock<std::mutex> bufLock(bufferMtx_);
71 if (consumerSurface_ == nullptr) {
72 DHLOGE("%{public}s: consumerSurface_ is nullptr.", DSCREEN_LOG_TAG);
73 return;
74 }
75 sptr<SurfaceBuffer> surfaceBuffer = nullptr;
76 syncFence_ = SyncFence::INVALID_FENCE;
77 int64_t timestamp = 0;
78 OHOS::Rect damage = {0, 0, 0, 0};
79 SurfaceError surfaceErr = consumerSurface_->AcquireBuffer(surfaceBuffer, syncFence_, timestamp, damage);
80 if (surfaceErr != SURFACE_ERROR_OK) {
81 DHLOGE("%{public}s: consumerSurface_ acquire buffer failed, errcode: %{public}" PRId32, DSCREEN_LOG_TAG,
82 surfaceErr);
83 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
84 return;
85 }
86 int32_t retcode = syncFence_->Wait(SURFACE_SYNC_FENCE_TIMEOUT);
87 if (retcode == -ETIME) {
88 DHLOGE("%{public}s: Sync fence wait timeout, retcode is %{public}" PRId32, DSCREEN_LOG_TAG, retcode);
89 return;
90 }
91 if (pHandler_ != nullptr) {
92 eventContent_.clear();
93 eventContent_ = imageSetDirtyPtr_->GetDamage();
94 }
95 std::vector<OHOS::Rect> damages = VecToDamage(eventContent_);
96 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
97 if (listener == nullptr) {
98 DHLOGE("%{public}s: Processor listener is null", DSCREEN_LOG_TAG);
99 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
100 return;
101 }
102 listener->OnDamageProcessDone(surfaceBuffer, damages);
103 consumerSurface_->ReleaseBuffer(surfaceBuffer, -1);
104 }
105
ConfigureEncoder(const VideoParam & configParam)106 int32_t ImageSourceEncoder::ConfigureEncoder(const VideoParam &configParam)
107 {
108 DHLOGI("%{public}s: ConfigureEncoder.", DSCREEN_LOG_TAG);
109 int32_t ret = InitVideoEncoder(configParam);
110 if (ret != DH_SUCCESS) {
111 DHLOGE("%{public}s: Init encoder failed ret: %{public}" PRId32, DSCREEN_LOG_TAG, ret);
112 return ret;
113 }
114
115 ret = SetEncoderFormat(configParam);
116 if (ret != DH_SUCCESS) {
117 DHLOGE("%{public}s: Set encoder format failed ret: %{public}" PRId32, DSCREEN_LOG_TAG, ret);
118 return ret;
119 }
120
121 encoderSurface_ = videoEncoder_->CreateInputSurface();
122 if (encoderSurface_ == nullptr) {
123 DHLOGE("%{public}s: Create encoder surface failed.", DSCREEN_LOG_TAG);
124 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
125 }
126 configParam_ = configParam;
127 ret = AddSurface();
128 if (ret != DH_SUCCESS) {
129 DHLOGE("%{public}s: Add surface failed ret: %{public}" PRId32, DSCREEN_LOG_TAG, ret);
130 std::unique_lock<std::mutex> bufLock(bufferMtx_);
131 consumerSurface_ = nullptr;
132 producerSurface_ = nullptr;
133 return ret;
134 }
135 return DH_SUCCESS;
136 }
137
AddSurface()138 int32_t ImageSourceEncoder::AddSurface()
139 {
140 DHLOGI("%{public}s: AddSurface.", DSCREEN_LOG_TAG);
141 std::unique_lock<std::mutex> bufLock(bufferMtx_);
142 consumerSurface_ = IConsumerSurface::Create();
143 if (consumerSurface_ == nullptr) {
144 DHLOGE("%{public}s: creat consumer surface failed.", DSCREEN_LOG_TAG);
145 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
146 }
147 sptr<IBufferProducer> producer = consumerSurface_->GetProducer();
148 if (producer == nullptr) {
149 DHLOGE("%{public}s: Creat producer surface failed.", DSCREEN_LOG_TAG);
150 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
151 }
152 producerSurface_ = Surface::CreateSurfaceAsProducer(producer);
153 if (producerSurface_ == nullptr) {
154 DHLOGE("%{public}s: Create preducer surface failed.", DSCREEN_LOG_TAG);
155 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
156 }
157 consumerBufferListener_ = new ConsumerBufferListener(shared_from_this());
158 consumerSurface_->RegisterConsumerListener(consumerBufferListener_);
159 return DH_SUCCESS;
160 }
161
GetConsumerSurface()162 sptr<Surface> ImageSourceEncoder::GetConsumerSurface()
163 {
164 DHLOGI("%{public}s: GetConsumerSurface.", DSCREEN_LOG_TAG);
165 return consumerSurface_;
166 }
167
GetInputSurface()168 sptr<Surface> &ImageSourceEncoder::GetInputSurface()
169 {
170 DHLOGI("%{public}s: GetInputSurface.", DSCREEN_LOG_TAG);
171 if (producerSurface_ == nullptr || !configParam_.GetPartialRefreshFlag()) {
172 return encoderSurface_;
173 }
174 return producerSurface_;
175 }
176
VecToDamage(std::vector<std::vector<int32_t>> eventContent)177 std::vector<OHOS::Rect> ImageSourceEncoder::VecToDamage(std::vector<std::vector<int32_t>> eventContent)
178 {
179 DHLOGI("%{public}s: VecToDamage.", DSCREEN_LOG_TAG);
180 std::vector<OHOS::Rect> damages;
181 for (auto item : eventContent) {
182 OHOS::Rect damage = {0, 0, 0, 0};
183 damage.x = item[0];
184 damage.y = item[1];
185 damage.w = item[2];
186 damage.h = item[3];
187 damages.push_back(damage);
188 }
189 return damages;
190 }
191
GetEncoderInputSurfaceBuffer()192 sptr<SurfaceBuffer> ImageSourceEncoder::GetEncoderInputSurfaceBuffer()
193 {
194 DHLOGI("%{public}s: GetEncoderInputSurfaceBuffer.", DSCREEN_LOG_TAG);
195 OHOS::BufferRequestConfig requestConfig;
196 requestConfig.width = static_cast<int32_t>(configParam_.GetVideoWidth());
197 requestConfig.height = static_cast<int32_t>(configParam_.GetVideoHeight());
198 requestConfig.usage = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA;
199 requestConfig.strideAlignment = STRIDE_ALIGNMENT;
200 requestConfig.format = GraphicPixelFormat::GRAPHIC_PIXEL_FMT_RGBA_8888;
201 requestConfig.timeout = 0;
202 sptr<OHOS::SurfaceBuffer> encoderSurfaceBuffer = nullptr;
203 int32_t releaseFence = -1;
204 SurfaceError surfaceErr = encoderSurface_->RequestBuffer(encoderSurfaceBuffer, releaseFence, requestConfig);
205 if (surfaceErr != GSERROR_OK || encoderSurfaceBuffer == nullptr) {
206 DHLOGE("%{public}s: RequestBuffer failed.", DSCREEN_LOG_TAG);
207 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
208 }
209 return encoderSurfaceBuffer;
210 }
211
FeedEncoderData(sptr<SurfaceBuffer> & surfaceBuffer)212 int32_t ImageSourceEncoder::FeedEncoderData(sptr<SurfaceBuffer> &surfaceBuffer)
213 {
214 sptr<OHOS::SurfaceBuffer> encoderSurfaceBuffer = GetEncoderInputSurfaceBuffer();
215 if (encoderSurfaceBuffer == nullptr) {
216 DHLOGE("Get encoder input producer surface buffer failed.");
217 return ERR_DH_SCREEN_CODEC_SURFACE_ERROR;
218 }
219 uint32_t screenDataSize = configParam_.GetVideoWidth() * configParam_.GetVideoHeight() * RGBA_CHROMA;
220 auto encoderSurfaceAddr = static_cast<uint8_t*>(encoderSurfaceBuffer->GetVirAddr());
221 auto surfaceAddr = static_cast<uint8_t*>(surfaceBuffer->GetVirAddr());
222 int32_t ret = memcpy_s(encoderSurfaceAddr, screenDataSize, surfaceAddr, screenDataSize);
223 if (ret != EOK) {
224 DHLOGE("%{public}s: surfaceBuffer memcpy_s run failed.", DSCREEN_LOG_TAG);
225 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
226 return ret;
227 }
228 BufferFlushConfig flushConfig = { {0, 0, encoderSurfaceBuffer->GetWidth(), encoderSurfaceBuffer->GetHeight()}, 0};
229 DHLOGI("%{public}s: FeedEncoderData to encoder.", DSCREEN_LOG_TAG);
230 SurfaceError surfaceErr = encoderSurface_->FlushBuffer(encoderSurfaceBuffer, -1, flushConfig);
231 if (surfaceErr != SURFACE_ERROR_OK) {
232 DHLOGE("%{public}s: encoderSurface_ flush buffer failed.", DSCREEN_LOG_TAG);
233 encoderSurface_->CancelBuffer(encoderSurfaceBuffer);
234 return surfaceErr;
235 }
236 return DH_SUCCESS;
237 }
238
ReleaseEncoder()239 int32_t ImageSourceEncoder::ReleaseEncoder()
240 {
241 DHLOGI("%{public}s: ReleaseEncoder.", DSCREEN_LOG_TAG);
242 if (videoEncoder_ == nullptr) {
243 DHLOGE("%{public}s: Encoder is null.", DSCREEN_LOG_TAG);
244 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
245 }
246
247 int32_t ret = videoEncoder_->Release();
248 if (ret != MediaAVCodec::AVCS_ERR_OK) {
249 DHLOGE("%{public}s: Release encoder failed.", DSCREEN_LOG_TAG);
250 return ERR_DH_SCREEN_CODEC_RELEASE_FAILED;
251 }
252 if (dscreenDbgItfPtr_ != nullptr) {
253 dscreenDbgItfPtr_->Release();
254 delete dscreenDbgItfPtr_;
255 dscreenDbgItfPtr_ = nullptr;
256 }
257 if (imageSetDirtyPtr_ != nullptr) {
258 delete imageSetDirtyPtr_;
259 imageSetDirtyPtr_ = nullptr;
260 }
261 if (pHandler_ != nullptr) {
262 dlclose(pHandler_);
263 }
264 encodeVideoCallback_ = nullptr;
265 videoEncoder_ = nullptr;
266 consumerBufferListener_ = nullptr;
267 return DH_SUCCESS;
268 }
269
StartEncoder()270 int32_t ImageSourceEncoder::StartEncoder()
271 {
272 DHLOGI("%{public}s: StartEncoder.", DSCREEN_LOG_TAG);
273 if (videoEncoder_ == nullptr) {
274 DHLOGE("%{public}s: Encoder is null.", DSCREEN_LOG_TAG);
275 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
276 }
277
278 int32_t ret = videoEncoder_->Prepare();
279 if (ret != MediaAVCodec::AVCS_ERR_OK) {
280 DHLOGE("%{public}s: Prepare encoder failed.", DSCREEN_LOG_TAG);
281 return ERR_DH_SCREEN_CODEC_PREPARE_FAILED;
282 }
283
284 ret = videoEncoder_->Start();
285 if (ret != MediaAVCodec::AVCS_ERR_OK) {
286 DHLOGE("%{public}s: Start encoder failed.", DSCREEN_LOG_TAG);
287 return ERR_DH_SCREEN_CODEC_START_FAILED;
288 }
289 InitDscreenDBG();
290 return DH_SUCCESS;
291 }
292
StopEncoder()293 int32_t ImageSourceEncoder::StopEncoder()
294 {
295 DHLOGI("%{public}s: StopEncoder.", DSCREEN_LOG_TAG);
296 if (videoEncoder_ == nullptr) {
297 DHLOGE("%{public}s: Encoder is null.", DSCREEN_LOG_TAG);
298 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
299 }
300
301 int32_t ret = videoEncoder_->Flush();
302 if (ret != MediaAVCodec::AVCS_ERR_OK) {
303 DHLOGE("%{public}s: Flush encoder failed.", DSCREEN_LOG_TAG);
304 }
305
306 ret = videoEncoder_->Stop();
307 if (ret != MediaAVCodec::AVCS_ERR_OK) {
308 DHLOGE("%{public}s: Stop encoder failed.", DSCREEN_LOG_TAG);
309 return ERR_DH_SCREEN_CODEC_STOP_FAILED;
310 }
311 std::unique_lock<std::mutex> bufLock(bufferMtx_);
312 if (consumerSurface_ != nullptr) {
313 ret = consumerSurface_->UnregisterConsumerListener();
314 if (ret != SURFACE_ERROR_OK) {
315 DHLOGE("Unregister Consumer Listener failed.");
316 }
317 consumerSurface_ = nullptr;
318 }
319 producerSurface_ = nullptr;
320 return DH_SUCCESS;
321 }
322
InitVideoEncoder(const VideoParam & configParam)323 int32_t ImageSourceEncoder::InitVideoEncoder(const VideoParam &configParam)
324 {
325 DHLOGI("%{public}s: InitVideoEncoder.", DSCREEN_LOG_TAG);
326 switch (configParam.GetCodecType()) {
327 case VIDEO_CODEC_TYPE_VIDEO_H264:
328 videoEncoder_ = MediaAVCodec::VideoEncoderFactory::CreateByMime(
329 std::string(MediaAVCodec::CodecMimeType::VIDEO_AVC));
330 break;
331 case VIDEO_CODEC_TYPE_VIDEO_H265:
332 videoEncoder_ = MediaAVCodec::VideoEncoderFactory::CreateByMime(
333 std::string(MediaAVCodec::CodecMimeType::VIDEO_HEVC));
334 break;
335 default:
336 DHLOGE("%{public}s: codecType is invalid!", DSCREEN_LOG_TAG);
337 videoEncoder_ = nullptr;
338 }
339
340 if (videoEncoder_ == nullptr) {
341 DHLOGE("%{public}s: Create videoEncoder failed.", DSCREEN_LOG_TAG);
342 return ERR_DH_SCREEN_TRANS_CREATE_CODEC_FAILED;
343 }
344
345 encodeVideoCallback_ = std::make_shared<ImageEncoderCallback>(shared_from_this());
346 int32_t ret = videoEncoder_->SetCallback(encodeVideoCallback_);
347 if (ret != MediaAVCodec::AVCS_ERR_OK) {
348 DHLOGE("%{public}s: Set codec callback failed.", DSCREEN_LOG_TAG);
349 return ERR_DH_SCREEN_CODEC_SET_CALLBACK_FAILED;
350 }
351
352 return DH_SUCCESS;
353 }
354
SetEncoderFormat(const VideoParam & configParam)355 int32_t ImageSourceEncoder::SetEncoderFormat(const VideoParam &configParam)
356 {
357 DHLOGI("%{public}s: SetEncoderFormat.", DSCREEN_LOG_TAG);
358 if (videoEncoder_ == nullptr) {
359 DHLOGE("%{public}s: Encoder is null.", DSCREEN_LOG_TAG);
360 return ERR_DH_SCREEN_TRANS_NULL_VALUE;
361 }
362
363 switch (configParam.GetCodecType()) {
364 case VIDEO_CODEC_TYPE_VIDEO_H264:
365 imageFormat_.PutStringValue("codec_mime", MediaAVCodec::CodecMimeType::VIDEO_AVC);
366 break;
367 case VIDEO_CODEC_TYPE_VIDEO_H265:
368 imageFormat_.PutStringValue("codec_mime", MediaAVCodec::CodecMimeType::VIDEO_HEVC);
369 break;
370 default:
371 DHLOGE("%{public}s: Codec type is invalid.", DSCREEN_LOG_TAG);
372 return ERR_DH_SCREEN_TRANS_ILLEGAL_PARAM;
373 }
374
375 imageFormat_.PutIntValue("pixel_format", static_cast<int32_t>(MediaAVCodec::VideoPixelFormat::RGBA));
376 imageFormat_.PutLongValue("max_input_size", MAX_YUV420_BUFFER_SIZE);
377 imageFormat_.PutIntValue("width", configParam.GetVideoWidth());
378 imageFormat_.PutIntValue("height", configParam.GetVideoHeight());
379 imageFormat_.PutDoubleValue("frame_rate", configParam.GetFps());
380
381 int32_t ret = videoEncoder_->Configure(imageFormat_);
382 if (ret != MediaAVCodec::AVCS_ERR_OK) {
383 DHLOGE("%{public}s: Configure encoder failed.", DSCREEN_LOG_TAG);
384 return ERR_DH_SCREEN_CODEC_CONFIGURE_FAILED;
385 }
386 return DH_SUCCESS;
387 }
388
OnError(MediaAVCodec::AVCodecErrorType errorType,int32_t errorCode)389 void ImageSourceEncoder::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode)
390 {
391 DHLOGI("%{public}s: Encoder error, errorType:%{public}" PRId32 ", errorCode:%{public}" PRId32, DSCREEN_LOG_TAG,
392 errorType, errorCode);
393 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
394 if (listener == nullptr) {
395 DHLOGE("%{public}s: Processor listener is null", DSCREEN_LOG_TAG);
396 return;
397 }
398 listener->OnProcessorStateNotify(errorCode);
399 }
400
OnOutputBufferAvailable(uint32_t index,MediaAVCodec::AVCodecBufferInfo info,MediaAVCodec::AVCodecBufferFlag flag,std::shared_ptr<Media::AVSharedMemory> buffer)401 void ImageSourceEncoder::OnOutputBufferAvailable(uint32_t index, MediaAVCodec::AVCodecBufferInfo info,
402 MediaAVCodec::AVCodecBufferFlag flag, std::shared_ptr<Media::AVSharedMemory> buffer)
403 {
404 DHLOGD("%{public}s: OnOutputBufferAvailable, receiv H264 data from encoder.", DSCREEN_LOG_TAG);
405 std::shared_ptr<IImageSourceProcessorListener> listener = imageProcessorListener_.lock();
406 if (listener == nullptr) {
407 DHLOGE("%{public}s: Processor listener is null", DSCREEN_LOG_TAG);
408 return;
409 }
410 if (videoEncoder_ == nullptr) {
411 DHLOGE("%{public}s: Encoder is null.", DSCREEN_LOG_TAG);
412 return;
413 }
414
415 encoderBufferInfo_ = info;
416 if (buffer == nullptr) {
417 DHLOGE("%{public}s: Buffer is null, index = %{public}d", DSCREEN_LOG_TAG, index);
418 return;
419 }
420
421 size_t dataSize = static_cast<size_t>(info.size);
422 if (dataSize == 0 || dataSize > DATA_BUFFER_MAX_SIZE) {
423 DHLOGE("%{public}s:OnOutputBufferAvailable params invalid, size: %{public}zu", DSCREEN_LOG_TAG, dataSize);
424 return;
425 }
426 auto dataBuf = std::make_shared<DataBuffer>(dataSize);
427 int32_t ret = memcpy_s(dataBuf->Data(), dataBuf->Capacity(), buffer->GetBase(), dataSize);
428 if (ret != EOK) {
429 DHLOGE("%{public}s: Copy data failed.", DSCREEN_LOG_TAG);
430 return;
431 }
432 dataBuf->SetDataType(VIDEO_FULL_SCREEN_DATA);
433 dataBuf->SetSize(dataSize);
434 listener->OnImageProcessDone(dataBuf);
435 ret = videoEncoder_->ReleaseOutputBuffer(index);
436 if (ret != MediaAVCodec::AVCS_ERR_OK) {
437 DHLOGE("%{public}s: videoEncoder ReleaseOutputBuffer failed.", DSCREEN_LOG_TAG);
438 }
439 }
440
OnInputBufferAvailable(uint32_t index,std::shared_ptr<Media::AVSharedMemory> buffer)441 void ImageSourceEncoder::OnInputBufferAvailable(uint32_t index, std::shared_ptr<Media::AVSharedMemory> buffer)
442 {
443 (void) index;
444 }
445
OnOutputFormatChanged(const Media::Format & format)446 void ImageSourceEncoder::OnOutputFormatChanged(const Media::Format &format)
447 {
448 (void) format;
449 }
450 } // namespace DistributedHardware
451 } // namespace OHOS
452