1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "image_effect_inner.h"
17
18 #include <cassert>
19 #include <securec.h>
20 #include <algorithm>
21 #include <sync_fence.h>
22
23 #include "metadata_helper.h"
24 #include "common_utils.h"
25 #include "filter_factory.h"
26 #include "image_sink_filter.h"
27 #include "image_source_filter.h"
28 #include "effect_surface_adapter.h"
29 #include "pipeline_core.h"
30 #include "json_helper.h"
31 #include "efilter_factory.h"
32 #include "external_loader.h"
33 #include "effect_context.h"
34 #include "colorspace_helper.h"
35 #include "memcpy_helper.h"
36 #include "render_task.h"
37 #include "render_environment.h"
38 #include "v1_1/buffer_handle_meta_key_type.h"
39 #include "effect_log.h"
40 #include "effect_trace.h"
41 #include "native_window.h"
42
43 #define RENDER_QUEUE_SIZE 8
44 #define COMMON_TASK_TAG 0
45 namespace OHOS {
46 namespace Media {
47 namespace Effect {
48 using namespace OHOS::HDI::Display::Graphic::Common;
49
50 enum class EffectState {
51 IDLE,
52 RUNNING,
53 };
54
55 const int STRUCT_IMAGE_EFFECT_CONSTANT = 1;
56 const int DESTRUCTOR_IMAGE_EFFECT_CONSTANT = 2;
57 const int VIDEO_SINK_FILTER_STATUS = 3;
58
59 class ImageEffect::Impl {
60 public:
Impl()61 Impl()
62 {
63 InitPipeline();
64 InitEffectContext();
65 }
66
67 void CreatePipeline(std::vector<std::shared_ptr<EFilter>> &efilters);
68 private:
69 void InitPipeline();
70 void InitEffectContext();
71
72 public:
73 std::unique_ptr<EffectSurfaceAdapter> surfaceAdapter_;
74 std::shared_ptr<PipelineCore> pipeline_;
75 std::shared_ptr<ImageSourceFilter> srcFilter_;
76 std::shared_ptr<ImageSinkFilter> sinkFilter_;
77 std::shared_ptr<EffectContext> effectContext_;
78 EffectState effectState_ = EffectState::IDLE;
79 };
80
InitPipeline()81 void ImageEffect::Impl::InitPipeline()
82 {
83 srcFilter_ = FilterFactory::Instance().CreateFilterWithType<ImageSourceFilter>(GET_FILTER_NAME(ImageSourceFilter));
84 sinkFilter_ = FilterFactory::Instance().CreateFilterWithType<ImageSinkFilter>(GET_FILTER_NAME(ImageSinkFilter));
85 CHECK_AND_RETURN(srcFilter_ != nullptr);
86 CHECK_AND_RETURN(sinkFilter_ != nullptr);
87 }
88
InitEffectContext()89 void ImageEffect::Impl::InitEffectContext()
90 {
91 effectContext_ = std::make_shared<EffectContext>();
92 effectContext_->memoryManager_ = std::make_shared<EffectMemoryManager>();
93 effectContext_->renderStrategy_ = std::make_shared<RenderStrategy>();
94 effectContext_->capNegotiate_ = std::make_shared<CapabilityNegotiate>();
95 effectContext_->renderEnvironment_ = std::make_shared<RenderEnvironment>();
96 effectContext_->colorSpaceManager_ = std::make_shared<ColorSpaceManager>();
97 effectContext_->metaInfoNegotiate_ = std::make_shared<EfilterMetaInfoNegotiate>();
98 }
99
CreatePipeline(std::vector<std::shared_ptr<EFilter>> & efilters)100 void ImageEffect::Impl::CreatePipeline(std::vector<std::shared_ptr<EFilter>> &efilters)
101 {
102 pipeline_ = std::make_shared<PipelineCore>();
103 pipeline_->Init(nullptr);
104
105 CHECK_AND_RETURN_LOG(srcFilter_ != nullptr, "srcFilter is null");
106 std::vector<Filter *> filtersToPipeline; // Note: Filters must be inserted in sequence.
107 filtersToPipeline.push_back(srcFilter_.get());
108 for (const auto &eFilter : efilters) {
109 filtersToPipeline.push_back(eFilter.get());
110 }
111 filtersToPipeline.push_back(sinkFilter_.get());
112
113 ErrorCode res = pipeline_->AddFilters(filtersToPipeline);
114 CHECK_AND_RETURN_LOG(res == ErrorCode::SUCCESS, "pipeline add filters fail! res=%{public}d", res);
115
116 res = pipeline_->LinkFilters(filtersToPipeline);
117 CHECK_AND_RETURN_LOG(res == ErrorCode::SUCCESS, "pipeline link filter fail! res=%{public}d", res);
118 }
119
120 struct EffectParameters {
EffectParametersOHOS::Media::Effect::EffectParameters121 EffectParameters(std::shared_ptr<EffectBuffer> &srcEffectBuffer, std::shared_ptr<EffectBuffer> &dstEffectBuffer,
122 std::map<ConfigType, Plugin::Any> &config, std::shared_ptr<EffectContext> &effectContext)
123 : srcEffectBuffer_(std::move(srcEffectBuffer)),
124 dstEffectBuffer_(std::move(dstEffectBuffer)),
125 config_(std::move(config)),
126 effectContext_(std::move(effectContext)) {};
127 std::shared_ptr<EffectBuffer> &&srcEffectBuffer_;
128 std::shared_ptr<EffectBuffer> &&dstEffectBuffer_;
129 std::map<ConfigType, Plugin::Any> &&config_;
130 std::shared_ptr<EffectContext> &&effectContext_;
131 };
132
133 enum class RunningType : int32_t {
134 DEFAULT = 0,
135 FOREGROUND = 1,
136 BACKGROUND = 2,
137 };
138
139 const std::vector<std::string> priorityEFilter_ = {
140 "Crop"
141 };
142 const std::unordered_map<std::string, ConfigType> configTypeTab_ = {
143 { "runningType", ConfigType::IPTYPE },
144 };
145 const std::unordered_map<int32_t, std::vector<IPType>> runningTypeTab_{
146 { std::underlying_type<RunningType>::type(RunningType::FOREGROUND), { IPType::CPU, IPType::GPU } },
147 { std::underlying_type<RunningType>::type(RunningType::BACKGROUND), { IPType::CPU } },
148 };
149
ImageEffect(const char * name)150 ImageEffect::ImageEffect(const char *name)
151 {
152 imageEffectFlag_ = STRUCT_IMAGE_EFFECT_CONSTANT;
153 impl_ = std::make_shared<Impl>();
154 if (name != nullptr) {
155 name_ = name;
156 }
157 ExternLoader::Instance()->InitExt();
158 ExtInitModule();
159
160 if (m_renderThread == nullptr) {
161 auto func = [this]() {};
162 m_renderThread = new RenderThread<>(RENDER_QUEUE_SIZE, func);
163 m_renderThread->Start();
164 auto task = std::make_shared<RenderTask<>>([this]() { this->InitEGLEnv(); }, COMMON_TASK_TAG,
165 RequestTaskId());
166 m_renderThread->AddTask(task);
167 task->Wait();
168 }
169 }
170
~ImageEffect()171 ImageEffect::~ImageEffect()
172 {
173 EFFECT_LOGI("ImageEffect destruct enter!");
174 imageEffectFlag_ = DESTRUCTOR_IMAGE_EFFECT_CONSTANT;
175 impl_->surfaceAdapter_ = nullptr;
176 m_renderThread->ClearTask();
177 auto task = std::make_shared<RenderTask<>>([this]() { this->DestroyEGLEnv(); }, COMMON_TASK_TAG,
178 RequestTaskId());
179 m_renderThread->AddTask(task);
180 task->Wait();
181 EFFECT_LOGI("ImageEffect destruct destroy egl env!");
182 ExtDeinitModule();
183 m_renderThread->Stop();
184 delete m_renderThread;
185
186 impl_->effectContext_->renderEnvironment_ = nullptr;
187 if (toProducerSurface_) {
188 auto res = toProducerSurface_->Disconnect();
189 EFFECT_LOGI("ImageEffect::~ImageEffect disconnect res=%{public}d, id=%{public}" PRIu64,
190 res, toProducerSurface_->GetUniqueId());
191 toProducerSurface_ = nullptr;
192 }
193 fromProducerSurface_ = nullptr;
194 impl_ = nullptr;
195 EFFECT_LOGI("ImageEffect destruct end!");
196 }
197
AddEFilter(const std::shared_ptr<EFilter> & efilter)198 void ImageEffect::AddEFilter(const std::shared_ptr<EFilter> &efilter)
199 {
200 std::unique_lock<std::mutex> lock(innerEffectMutex);
201 auto priorityEFilter = std::find_if(priorityEFilter_.begin(), priorityEFilter_.end(),
202 [&efilter](const std::string &name) { return name.compare(efilter->GetName()) == 0; });
203 if (priorityEFilter == priorityEFilter_.end()) {
204 efilters_.emplace_back(efilter);
205 } else {
206 auto result =
207 std::find_if(efilters_.rbegin(), efilters_.rend(), [&priorityEFilter](std::shared_ptr<EFilter> &efilter) {
208 return priorityEFilter->compare(efilter->GetName()) == 0;
209 });
210 if (result == efilters_.rend()) {
211 efilters_.insert(efilters_.begin(), efilter);
212 } else {
213 efilters_.insert(result.base(), efilter);
214 }
215 }
216
217 impl_->CreatePipeline(efilters_);
218 }
219
InsertEFilter(const std::shared_ptr<EFilter> & efilter,uint32_t index)220 ErrorCode ImageEffect::InsertEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index)
221 {
222 ErrorCode res = Effect::InsertEFilter(efilter, index);
223 if (res == ErrorCode::SUCCESS) {
224 impl_->CreatePipeline(efilters_);
225 }
226 return res;
227 }
228
RemoveEFilter(const std::shared_ptr<EFilter> & efilter)229 void ImageEffect::RemoveEFilter(const std::shared_ptr<EFilter> &efilter)
230 {
231 Effect::RemoveEFilter(efilter);
232 impl_->CreatePipeline(efilters_);
233 }
234
RemoveEFilter(uint32_t index)235 ErrorCode ImageEffect::RemoveEFilter(uint32_t index)
236 {
237 ErrorCode res = Effect::RemoveEFilter(index);
238 if (res == ErrorCode::SUCCESS) {
239 impl_->CreatePipeline(efilters_);
240 }
241 return res;
242 }
243
ReplaceEFilter(const std::shared_ptr<EFilter> & efilter,uint32_t index)244 ErrorCode ImageEffect::ReplaceEFilter(const std::shared_ptr<EFilter> &efilter, uint32_t index)
245 {
246 std::unique_lock<std::mutex> lock(innerEffectMutex);
247 ErrorCode res = Effect::ReplaceEFilter(efilter, index);
248 if (res == ErrorCode::SUCCESS) {
249 impl_->CreatePipeline(efilters_);
250 }
251 return res;
252 }
253
RequestTaskId()254 unsigned long int ImageEffect::RequestTaskId()
255 {
256 return m_currentTaskId.fetch_add(1);
257 }
258
SetInputPixelMap(PixelMap * pixelMap)259 ErrorCode ImageEffect::SetInputPixelMap(PixelMap* pixelMap)
260 {
261 std::unique_lock<std::mutex> lock(innerEffectMutex);
262 EFFECT_LOGD("ImageEffect::SetInputPixelMap");
263 CHECK_AND_RETURN_RET_LOG(pixelMap != nullptr, ErrorCode::ERR_INVALID_SRC_PIXELMAP, "invalid source pixelMap");
264 impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
265
266 ClearDataInfo(inDateInfo_);
267 inDateInfo_.dataType_ = DataType::PIXEL_MAP;
268 inDateInfo_.pixelMap_ = pixelMap;
269 return ErrorCode::SUCCESS;
270 }
271
ConfigSourceFilter(std::shared_ptr<ImageSourceFilter> & srcFilter,std::shared_ptr<EffectBuffer> & srcBuffer,std::shared_ptr<EffectContext> & context)272 ErrorCode ConfigSourceFilter(std::shared_ptr<ImageSourceFilter> &srcFilter, std::shared_ptr<EffectBuffer> &srcBuffer,
273 std::shared_ptr<EffectContext> &context)
274 {
275 CHECK_AND_RETURN_RET_LOG(srcFilter != nullptr, ErrorCode::ERR_INPUT_NULL, "srcFilter is null");
276
277 ErrorCode res = srcFilter->SetSource(srcBuffer, context);
278 FALSE_RETURN_MSG_E(res == ErrorCode::SUCCESS, res, "set source fail! res=%{public}d", res);
279
280 return ErrorCode::SUCCESS;
281 }
282
ConfigSinkFilter(std::shared_ptr<ImageSinkFilter> & sinkFilter,std::shared_ptr<EffectBuffer> & sinkBuffer)283 ErrorCode ConfigSinkFilter(std::shared_ptr<ImageSinkFilter> &sinkFilter, std::shared_ptr<EffectBuffer> &sinkBuffer)
284 {
285 CHECK_AND_RETURN_RET_LOG(sinkFilter != nullptr, ErrorCode::ERR_INPUT_NULL, "sinkFilter is null");
286
287 ErrorCode res = sinkFilter->SetSink(sinkBuffer);
288 FALSE_RETURN_MSG_E(res == ErrorCode::SUCCESS, res, "set sink fail! res=%{public}d", res);
289
290 return ErrorCode::SUCCESS;
291 }
292
GetConfigIPTypes(const std::map<ConfigType,Plugin::Any> & config,std::vector<IPType> & configIPTypes)293 void GetConfigIPTypes(const std::map<ConfigType, Plugin::Any> &config, std::vector<IPType> &configIPTypes)
294 {
295 auto it = config.find(ConfigType::IPTYPE);
296 if (it == config.end()) {
297 EFFECT_LOGE("ipType config not set! use default config.");
298 configIPTypes = { IPType::CPU, IPType::GPU };
299 return;
300 }
301
302 ErrorCode result = CommonUtils::ParseAny(it->second, configIPTypes);
303 if (result == ErrorCode::SUCCESS) {
304 return;
305 }
306 EFFECT_LOGE("parse ipType fail! use default config.");
307 configIPTypes = { IPType::CPU, IPType::GPU };
308 }
309
ChooseIPType(const std::shared_ptr<EffectBuffer> & srcEffectBuffer,const std::shared_ptr<EffectContext> & context,const std::map<ConfigType,Plugin::Any> & config,IPType & runningIPType)310 ErrorCode ChooseIPType(const std::shared_ptr<EffectBuffer> &srcEffectBuffer,
311 const std::shared_ptr<EffectContext> &context, const std::map<ConfigType, Plugin::Any> &config,
312 IPType &runningIPType)
313 {
314 std::vector<IPType> configIPTypes;
315 GetConfigIPTypes(config, configIPTypes);
316
317 runningIPType = IPType::DEFAULT;
318 IPType priorityIPType = IPType::GPU;
319 IEffectFormat effectFormat = srcEffectBuffer->bufferInfo_->formatType_;
320 const std::vector<std::shared_ptr<Capability>> &caps = context->capNegotiate_->GetCapabilityList();
321 for (const auto &capability : caps) {
322 if (capability == nullptr || capability->pixelFormatCap_ == nullptr) {
323 continue;
324 }
325 std::map<IEffectFormat, std::vector<IPType>> &formats = capability->pixelFormatCap_->formats;
326 if (runningIPType == IPType::GPU) {
327 effectFormat = IEffectFormat::RGBA8888;
328 }
329
330 auto it = formats.find(effectFormat);
331 if (it == formats.end()) {
332 EFFECT_LOGE("effectFormat not support! effectFormat=%{public}d, name=%{public}s",
333 effectFormat, capability->name_.c_str());
334 return ErrorCode::SUCCESS;
335 }
336
337 std::vector<IPType> &ipTypes = it->second;
338 if (std::find(configIPTypes.begin(), configIPTypes.end(), priorityIPType) != configIPTypes.end() &&
339 std::find(ipTypes.begin(), ipTypes.end(), priorityIPType) != ipTypes.end()) {
340 runningIPType = IPType::GPU;
341 } else {
342 if (runningIPType == IPType::DEFAULT) {
343 runningIPType = IPType::CPU;
344 }
345 return ErrorCode::SUCCESS;
346 }
347 }
348
349 return ErrorCode::SUCCESS;
350 }
351
StartPipelineInner(std::shared_ptr<PipelineCore> & pipeline,const EffectParameters & effectParameters,unsigned long int taskId,RenderThread<> * thread)352 ErrorCode StartPipelineInner(std::shared_ptr<PipelineCore> &pipeline, const EffectParameters &effectParameters,
353 unsigned long int taskId, RenderThread<> *thread)
354 {
355 if (thread == nullptr) {
356 EFFECT_LOGE("pipeline Prepare fail! render thread is nullptr");
357 return ErrorCode::ERR_INVALID_OPERATION;
358 }
359 auto prom = std::make_shared<std::promise<ErrorCode>>();
360 std::future<ErrorCode> fut = prom->get_future();
361 auto task = std::make_shared<RenderTask<>>([pipeline, &effectParameters, &prom]() {
362 ErrorCode res = pipeline->Prepare();
363 if (res != ErrorCode::SUCCESS) {
364 EFFECT_LOGE("pipeline Prepare fail! res=%{public}d", res);
365 prom->set_value(res);
366 return;
367 }
368
369 res = ColorSpaceHelper::ConvertColorSpace(effectParameters.srcEffectBuffer_, effectParameters.effectContext_);
370 if (res != ErrorCode::SUCCESS) {
371 EFFECT_LOGE("StartPipelineInner:ConvertColorSpace fail! res=%{public}d", res);
372 prom->set_value(res);
373 return;
374 }
375
376 IPType runningIPType;
377 res = ChooseIPType(effectParameters.srcEffectBuffer_, effectParameters.effectContext_, effectParameters.config_,
378 runningIPType);
379 if (res != ErrorCode::SUCCESS) {
380 EFFECT_LOGE("choose running ip type fail! res=%{public}d", res);
381 prom->set_value(res);
382 return;
383 }
384 effectParameters.effectContext_->ipType_ = runningIPType;
385 effectParameters.effectContext_->memoryManager_->SetIPType(runningIPType);
386
387 res = pipeline->Start();
388 prom->set_value(res);
389 if (res != ErrorCode::SUCCESS) {
390 EFFECT_LOGE("pipeline start fail! res=%{public}d", res);
391 return;
392 }
393 }, 0, taskId);
394 thread->AddTask(task);
395 task->Wait();
396 ErrorCode res = fut.get();
397 return res;
398 }
399
StartPipeline(std::shared_ptr<PipelineCore> & pipeline,const EffectParameters & effectParameters,unsigned long int taskId,RenderThread<> * thread)400 ErrorCode StartPipeline(std::shared_ptr<PipelineCore> &pipeline, const EffectParameters &effectParameters,
401 unsigned long int taskId, RenderThread<> *thread)
402 {
403 effectParameters.effectContext_->renderStrategy_->Init(effectParameters.srcEffectBuffer_,
404 effectParameters.dstEffectBuffer_);
405 effectParameters.effectContext_->colorSpaceManager_->Init(effectParameters.srcEffectBuffer_,
406 effectParameters.dstEffectBuffer_);
407 effectParameters.effectContext_->memoryManager_->Init(effectParameters.srcEffectBuffer_,
408 effectParameters.dstEffectBuffer_);
409 ErrorCode res = StartPipelineInner(pipeline, effectParameters, taskId, thread);
410 effectParameters.effectContext_->memoryManager_->Deinit();
411 effectParameters.effectContext_->colorSpaceManager_->Deinit();
412 effectParameters.effectContext_->renderStrategy_->Deinit();
413 effectParameters.effectContext_->capNegotiate_->ClearNegotiateResult();
414 return res;
415 }
416
Start()417 ErrorCode ImageEffect::Start()
418 {
419 switch (inDateInfo_.dataType_) {
420 case DataType::PIXEL_MAP:
421 case DataType::SURFACE_BUFFER:
422 case DataType::URI:
423 case DataType::PATH:
424 case DataType::PICTURE: {
425 impl_->effectState_ = EffectState::RUNNING;
426 ErrorCode res = this->Render();
427 Stop();
428 return res;
429 }
430 case DataType::SURFACE:
431 impl_->effectState_ = EffectState::RUNNING;
432 if (impl_->surfaceAdapter_) {
433 impl_->surfaceAdapter_->ConsumerRequestCpuAccess(true);
434 }
435 break;
436 default:
437 EFFECT_LOGE("Not set input data!");
438 return ErrorCode::ERR_NOT_SET_INPUT_DATA;
439 }
440 return ErrorCode::SUCCESS;
441 }
442
Stop()443 void ImageEffect::Stop()
444 {
445 std::unique_lock<std::mutex> lock(innerEffectMutex);
446 impl_->effectState_ = EffectState::IDLE;
447 if (impl_->surfaceAdapter_) {
448 impl_->surfaceAdapter_->ConsumerRequestCpuAccess(false);
449 }
450 impl_->effectContext_->memoryManager_->ClearMemory();
451 }
452
SetInputSurfaceBuffer(OHOS::SurfaceBuffer * surfaceBuffer)453 ErrorCode ImageEffect::SetInputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer)
454 {
455 CHECK_AND_RETURN_RET_LOG(surfaceBuffer != nullptr, ErrorCode::ERR_INVALID_SRC_SURFACEBUFFER,
456 "invalid source surface buffer");
457
458 ClearDataInfo(inDateInfo_);
459 inDateInfo_.dataType_ = DataType::SURFACE_BUFFER;
460 inDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = surfaceBuffer;
461
462 return ErrorCode::SUCCESS;
463 }
464
SetOutputSurfaceBuffer(OHOS::SurfaceBuffer * surfaceBuffer)465 ErrorCode ImageEffect::SetOutputSurfaceBuffer(OHOS::SurfaceBuffer *surfaceBuffer)
466 {
467 ClearDataInfo(outDateInfo_);
468 if (surfaceBuffer == nullptr) {
469 EFFECT_LOGI("SetOutputSurfaceBuffer: surfaceBuffer set to null!");
470 return ErrorCode::SUCCESS;
471 }
472
473 outDateInfo_.dataType_ = DataType::SURFACE_BUFFER;
474 outDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = surfaceBuffer;
475
476 return ErrorCode::SUCCESS;
477 }
478
SetInputUri(const std::string & uri)479 ErrorCode ImageEffect::SetInputUri(const std::string &uri)
480 {
481 EFFECT_LOGD("ImageEffect::SetInputUri");
482 if (!CommonUtils::EndsWithJPG(uri)) {
483 EFFECT_LOGE("SetInputUri: file type is not support! only support jpg/jpeg.");
484 return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
485 }
486 ClearDataInfo(inDateInfo_);
487 inDateInfo_.dataType_ = DataType::URI;
488 inDateInfo_.uri_ = std::move(uri);
489
490 return ErrorCode::SUCCESS;
491 }
492
SetOutputUri(const std::string & uri)493 ErrorCode ImageEffect::SetOutputUri(const std::string &uri)
494 {
495 EFFECT_LOGD("ImageEffect::SetOutputUri");
496 if (uri.empty()) {
497 EFFECT_LOGI("SetOutputUri: uri set to null!");
498 ClearDataInfo(outDateInfo_);
499 return ErrorCode::SUCCESS;
500 }
501
502 if (!CommonUtils::EndsWithJPG(uri)) {
503 EFFECT_LOGE("SetOutputUri: file type is not support! only support jpg/jpeg.");
504 return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
505 }
506 ClearDataInfo(outDateInfo_);
507 outDateInfo_.dataType_ = DataType::URI;
508 outDateInfo_.uri_ = std::move(uri);
509
510 return ErrorCode::SUCCESS;
511 }
512
SetInputPath(const std::string & path)513 ErrorCode ImageEffect::SetInputPath(const std::string &path)
514 {
515 EFFECT_LOGD("ImageEffect::SetInputPath");
516 if (!CommonUtils::EndsWithJPG(path)) {
517 EFFECT_LOGE("SetInputPath: file type is not support! only support jpg/jpeg.");
518 return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
519 }
520 ClearDataInfo(inDateInfo_);
521 inDateInfo_.dataType_ = DataType::PATH;
522 inDateInfo_.path_ = std::move(path);
523
524 return ErrorCode::SUCCESS;
525 }
526
SetOutputPath(const std::string & path)527 ErrorCode ImageEffect::SetOutputPath(const std::string &path)
528 {
529 EFFECT_LOGD("ImageEffect::SetOutputPath");
530 if (path.empty()) {
531 EFFECT_LOGI("SetOutputPath: path set to null!");
532 ClearDataInfo(outDateInfo_);
533 return ErrorCode::SUCCESS;
534 }
535
536 if (!CommonUtils::EndsWithJPG(path)) {
537 EFFECT_LOGE("SetOutputPath: file type is not support! only support jpg/jpeg.");
538 return ErrorCode::ERR_FILE_TYPE_NOT_SUPPORT;
539 }
540 ClearDataInfo(outDateInfo_);
541 outDateInfo_.dataType_ = DataType::PATH;
542 outDateInfo_.path_ = std::move(path);
543
544 return ErrorCode::SUCCESS;
545 }
546
CheckToRenderPara(std::shared_ptr<EffectBuffer> & srcEffectBuffer,std::shared_ptr<EffectBuffer> & dstEffectBuffer)547 ErrorCode CheckToRenderPara(std::shared_ptr<EffectBuffer> &srcEffectBuffer,
548 std::shared_ptr<EffectBuffer> &dstEffectBuffer)
549 {
550 CHECK_AND_RETURN_RET_LOG(srcEffectBuffer != nullptr, ErrorCode::ERR_PARSE_FOR_EFFECT_BUFFER_FAIL,
551 "invalid srcEffectBuffer");
552
553 // allow developers not to set the out parameter.
554 if (dstEffectBuffer == nullptr) {
555 return ErrorCode::SUCCESS;
556 }
557
558 CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->bufferInfo_ != nullptr && dstEffectBuffer->bufferInfo_ != nullptr,
559 ErrorCode::ERR_BUFFER_INFO_NULL,
560 "buffer info is null! srcBufferInfo=%{public}d, dstBufferInfo=%{public}d",
561 srcEffectBuffer->bufferInfo_ == nullptr, dstEffectBuffer->bufferInfo_ == nullptr);
562 CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->extraInfo_ != nullptr && dstEffectBuffer->extraInfo_ != nullptr,
563 ErrorCode::ERR_EXTRA_INFO_NULL,
564 "extra info is null! srcExtraInfo=%{public}d, dstExtraInfo=%{public}d",
565 srcEffectBuffer->extraInfo_ == nullptr, dstEffectBuffer->extraInfo_ == nullptr);
566
567 // input and output type is same or not.
568 DataType srcDataType = srcEffectBuffer->extraInfo_->dataType;
569 DataType dtsDataType = dstEffectBuffer->extraInfo_->dataType;
570 std::function<bool(DataType, DataType)> dataTypeCheckFunc = [](DataType srcDataType, DataType dstDataType) {
571 if (srcDataType == dstDataType) {
572 return true;
573 }
574 std::vector<std::pair<DataType, DataType>> extraSupportTab = {
575 { DataType::PIXEL_MAP, DataType::NATIVE_WINDOW },
576 { DataType::PICTURE, DataType::NATIVE_WINDOW },
577 };
578 return extraSupportTab.end() != std::find_if(extraSupportTab.begin(), extraSupportTab.end(),
579 [&srcDataType, &dstDataType](const std::pair<DataType, DataType> &data) {
580 return data.first == srcDataType && data.second == dstDataType;
581 });
582 };
583 CHECK_AND_RETURN_RET_LOG(dataTypeCheckFunc(srcDataType, dtsDataType), ErrorCode::ERR_NOT_SUPPORT_DIFF_DATATYPE,
584 "not supported dataType. srcDataType=%{public}d, dstDataType=%{public}d", srcDataType, dtsDataType);
585
586 // color space is same or not.
587 if (srcDataType == DataType::PIXEL_MAP && dtsDataType != DataType::NATIVE_WINDOW) {
588 // the format for pixel map is same or not.
589 CHECK_AND_RETURN_RET_LOG(srcEffectBuffer->bufferInfo_->formatType_ == dstEffectBuffer->bufferInfo_->formatType_,
590 ErrorCode::ERR_NOT_SUPPORT_DIFF_FORMAT,
591 "not support different format. srcFormat=%{public}d, dstFormat=%{public}d",
592 srcEffectBuffer->bufferInfo_->formatType_, dstEffectBuffer->bufferInfo_->formatType_);
593
594 // color space is same or not.
595 EffectColorSpace srcColorSpace = srcEffectBuffer->bufferInfo_->colorSpace_;
596 EffectColorSpace dstColorSpace = dstEffectBuffer->bufferInfo_->colorSpace_;
597 bool isSrcHdr = ColorSpaceHelper::IsHdrColorSpace(srcColorSpace);
598 bool isDstHdr = ColorSpaceHelper::IsHdrColorSpace(dstColorSpace);
599 CHECK_AND_RETURN_RET_LOG(isSrcHdr == isDstHdr, ErrorCode::ERR_NOT_SUPPORT_INPUT_OUTPUT_COLORSPACE,
600 "not support different colorspace. src=%{public}d, dst=%{public}d", srcColorSpace, dstColorSpace);
601 }
602
603 return ErrorCode::SUCCESS;
604 }
605
Render()606 ErrorCode ImageEffect::Render()
607 {
608 if (efilters_.empty()) {
609 EFFECT_LOGW("efilters is empty!");
610 return ErrorCode::ERR_NOT_FILTERS_WITH_RENDER;
611 }
612
613 std::shared_ptr<EffectBuffer> srcEffectBuffer = nullptr;
614 std::shared_ptr<EffectBuffer> dstEffectBuffer = nullptr;
615 ErrorCode res = LockAll(srcEffectBuffer, dstEffectBuffer);
616 if (res != ErrorCode::SUCCESS) {
617 UnLockAll();
618 return res;
619 }
620
621 res = CheckToRenderPara(srcEffectBuffer, dstEffectBuffer);
622 if (res != ErrorCode::SUCCESS) {
623 UnLockAll();
624 return res;
625 }
626
627 std::shared_ptr<ImageSourceFilter> &sourceFilter = impl_->srcFilter_;
628 res = ConfigSourceFilter(sourceFilter, srcEffectBuffer, impl_->effectContext_);
629 if (res != ErrorCode::SUCCESS) {
630 UnLockAll();
631 return res;
632 }
633
634 std::shared_ptr<ImageSinkFilter> &sinkFilter = impl_->sinkFilter_;
635 res = ConfigSinkFilter(sinkFilter, dstEffectBuffer);
636 if (res != ErrorCode::SUCCESS) {
637 UnLockAll();
638 return res;
639 }
640 if (dstEffectBuffer != nullptr) {
641 impl_->effectContext_->renderEnvironment_->SetOutputType(dstEffectBuffer->extraInfo_->dataType);
642 } else {
643 impl_->effectContext_->renderEnvironment_->SetOutputType(srcEffectBuffer->extraInfo_->dataType);
644 }
645
646 EffectParameters effectParameters(srcEffectBuffer, dstEffectBuffer, config_, impl_->effectContext_);
647 res = StartPipeline(impl_->pipeline_, effectParameters, RequestTaskId(), m_renderThread);
648 if (res != ErrorCode::SUCCESS) {
649 EFFECT_LOGE("StartPipeline fail! res=%{public}d", res);
650 UnLockAll();
651 return res;
652 }
653
654 UnLockAll();
655 return res;
656 }
657
Save(EffectJsonPtr & res)658 ErrorCode ImageEffect::Save(EffectJsonPtr &res)
659 {
660 EffectJsonPtr effect = JsonHelper::CreateArray();
661 for (auto it = efilters_.begin(); it != efilters_.end(); it++) {
662 EffectJsonPtr data = JsonHelper::CreateObject();
663 std::shared_ptr<EFilter> efilter = *it;
664 efilter->Save(data);
665 effect->Add(data);
666 }
667
668 EffectJsonPtr info = JsonHelper::CreateObject();
669 info->Put("filters", effect);
670 info->Put("name", name_);
671 if (extraInfo_ != nullptr) {
672 extraInfo_->Replace("imageEffect", info);
673 res = extraInfo_;
674 } else {
675 res->Put("imageEffect", info);
676 }
677 return ErrorCode::SUCCESS;
678 }
679
Restore(std::string & info)680 std::shared_ptr<ImageEffect> ImageEffect::Restore(std::string &info)
681 {
682 const EffectJsonPtr root = JsonHelper::ParseJsonData(info);
683 CHECK_AND_RETURN_RET_LOG(root->HasElement("imageEffect"), nullptr, "Restore: no imageEffect");
684 const EffectJsonPtr &imageInfo = root->GetElement("imageEffect");
685 CHECK_AND_RETURN_RET_LOG(imageInfo != nullptr, nullptr, "Restore: imageInfo is null!");
686 CHECK_AND_RETURN_RET_LOG(imageInfo->HasElement("name"), nullptr, "Restore: imageEffect no name");
687 std::string effectName = imageInfo->GetString("name");
688 CHECK_AND_RETURN_RET_LOG(!effectName.empty(), nullptr, "Restore: imageEffect get name failed");
689
690 CHECK_AND_RETURN_RET_LOG(imageInfo->HasElement("filters"), nullptr, "Restore: imageEffect no filters");
691 std::vector<EffectJsonPtr> efiltersInfo = imageInfo->GetArray("filters");
692 CHECK_AND_RETURN_RET_LOG(!efiltersInfo.empty(), nullptr, "Restore: filters not array");
693
694 std::shared_ptr<ImageEffect> imageEffect = std::make_unique<ImageEffect>(effectName.c_str());
695 for (auto &efilterInfo : efiltersInfo) {
696 std::string name = efilterInfo->GetString("name");
697 CHECK_AND_CONTINUE_LOG(!name.empty(), "Restore: [name] not exist");
698 std::shared_ptr<EFilter> efilter = EFilterFactory::Instance()->Restore(name, efilterInfo, nullptr);
699 imageEffect->AddEFilter(efilter);
700 }
701 return imageEffect;
702 }
703
SetOutputPixelMap(PixelMap * pixelMap)704 ErrorCode ImageEffect::SetOutputPixelMap(PixelMap* pixelMap)
705 {
706 std::unique_lock<std::mutex> lock(innerEffectMutex);
707 EFFECT_LOGD("ImageEffect::SetOutputPixelMap");
708 ClearDataInfo(outDateInfo_);
709 if (pixelMap == nullptr) {
710 EFFECT_LOGI("SetOutputPixelMap: pixelMap set to null!");
711 return ErrorCode::SUCCESS;
712 }
713
714 outDateInfo_.dataType_ = DataType::PIXEL_MAP;
715 outDateInfo_.pixelMap_ = pixelMap;
716
717 return ErrorCode::SUCCESS;
718 }
719
SetOutputSurface(sptr<Surface> & surface)720 ErrorCode ImageEffect::SetOutputSurface(sptr<Surface>& surface)
721 {
722 std::unique_lock<std::mutex> lock(innerEffectMutex);
723 if (surface == nullptr) {
724 EFFECT_LOGE("surface is null.");
725 return ErrorCode::ERR_INPUT_NULL;
726 }
727 outDateInfo_.dataType_ = DataType::SURFACE;
728 toProducerSurface_ = surface;
729
730 if (impl_->surfaceAdapter_ == nullptr) {
731 impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
732 }
733 impl_->surfaceAdapter_->SetOutputSurfaceDefaultUsage(toProducerSurface_->GetDefaultUsage());
734
735 toProducerSurface_->SetTransform(GRAPHIC_ROTATE_BUTT);
736 return ErrorCode::SUCCESS;
737 }
738
UpdateProducerSurfaceInfo()739 void ImageEffect::UpdateProducerSurfaceInfo()
740 {
741 if (impl_->surfaceAdapter_ == nullptr) {
742 EFFECT_LOGE("impl surfaceAdapter is nullptr!");
743 return;
744 }
745 auto transform = impl_->surfaceAdapter_->GetTransform();
746 if (transform == toProducerTransform_) {
747 return;
748 }
749 toProducerTransform_ = transform;
750 EFFECT_LOGI("Set toProducerSurface transform %{public}d, GRAPHIC_ROTATE_270: %{public}d",
751 transform, GRAPHIC_ROTATE_270);
752
753 if (toProducerSurface_ == nullptr) {
754 EFFECT_LOGE("toProducerSurface_ is nullptr!");
755 return;
756 }
757 toProducerSurface_->SetTransform(transform);
758 }
759
ConsumerBufferWithGPU(sptr<SurfaceBuffer> & buffer)760 void ImageEffect::ConsumerBufferWithGPU(sptr<SurfaceBuffer>& buffer)
761 {
762 inDateInfo_.surfaceBufferInfo_.surfaceBuffer_ = buffer;
763 GraphicTransformType transform = impl_->surfaceAdapter_->GetTransform();
764 buffer->SetSurfaceBufferTransform(transform);
765 if (impl_->effectState_ == EffectState::RUNNING) {
766 impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
767 this->Render();
768 } else {
769 auto task = std::make_shared<RenderTask<>>([buffer, this, transform]() {
770 if (impl_->effectContext_->renderEnvironment_->GetEGLStatus() != EGLStatus::READY) {
771 impl_->effectContext_->renderEnvironment_->Init();
772 impl_->effectContext_->renderEnvironment_->Prepare();
773 }
774 int tex = GLUtils::CreateTextureFromSurfaceBuffer(buffer);
775 impl_->effectContext_->renderEnvironment_->UpdateCanvas();
776 impl_->effectContext_->renderEnvironment_->DrawFrame(tex, transform);
777 }, COMMON_TASK_TAG, RequestTaskId());
778 m_renderThread->AddTask(task);
779 task->Wait();
780 }
781 }
782
MemoryCopyForSurfaceBuffer(sptr<SurfaceBuffer> & buffer,OHOS::sptr<SurfaceBuffer> & outBuffer)783 void MemoryCopyForSurfaceBuffer(sptr<SurfaceBuffer> &buffer, OHOS::sptr<SurfaceBuffer> &outBuffer)
784 {
785 CopyInfo src = {
786 .bufferInfo = {
787 .width_ = static_cast<uint32_t>(buffer->GetWidth()),
788 .height_ = static_cast<uint32_t>(buffer->GetHeight()),
789 .len_ = buffer->GetSize(),
790 .formatType_ = CommonUtils::SwitchToEffectFormat((GraphicPixelFormat)buffer->GetFormat()),
791 .rowStride_ = static_cast<uint32_t>(buffer->GetStride()),
792 },
793 .data = static_cast<uint8_t *>(buffer->GetVirAddr()),
794 };
795 CopyInfo dst = {
796 .bufferInfo = {
797 .width_ = static_cast<uint32_t>(outBuffer->GetWidth()),
798 .height_ = static_cast<uint32_t>(outBuffer->GetHeight()),
799 .len_ = outBuffer->GetSize(),
800 .formatType_ = CommonUtils::SwitchToEffectFormat((GraphicPixelFormat)outBuffer->GetFormat()),
801 .rowStride_ = static_cast<uint32_t>(outBuffer->GetStride()),
802 },
803 .data = static_cast<uint8_t *>(outBuffer->GetVirAddr()),
804 };
805 MemcpyHelper::CopyData(src, dst);
806 }
807
IsSurfaceBufferHebc(sptr<SurfaceBuffer> & buffer)808 bool IsSurfaceBufferHebc(sptr<SurfaceBuffer> &buffer)
809 {
810 V1_1::BufferHandleAttrKey key = V1_1::BufferHandleAttrKey::ATTRKEY_ACCESS_TYPE;
811 std::vector<uint8_t> values;
812 auto res = buffer->GetMetadata(key, values);
813 CHECK_AND_RETURN_RET(res == 0, false);
814
815 V1_1::HebcAccessType hebcAccessType = V1_1::HebcAccessType::HEBC_ACCESS_UNINIT;
816 res = MetadataHelper::ConvertVecToMetadata(values, hebcAccessType);
817 CHECK_AND_RETURN_RET(res == 0, false);
818
819 if (hebcAccessType == V1_1::HEBC_ACCESS_HW_ONLY) {
820 EFFECT_LOGD("IsSurfaceBufferHebc: surface buffer is Hebc data!");
821 return true;
822 }
823 return false;
824 }
825
SetSurfaceBufferHebcAccessType(sptr<SurfaceBuffer> & buffer,V1_1::HebcAccessType hebcAccessType)826 void SetSurfaceBufferHebcAccessType(sptr<SurfaceBuffer> &buffer, V1_1::HebcAccessType hebcAccessType)
827 {
828 std::vector<uint8_t> values;
829 auto res = MetadataHelper::ConvertMetadataToVec(hebcAccessType, values);
830 CHECK_AND_RETURN_LOG(res == 0, "SetSurfaceBufferHebcAccessType: ConvertVecToMetadata fail! res=%{public}d", res);
831
832 V1_1::BufferHandleAttrKey key = V1_1::BufferHandleAttrKey::ATTRKEY_ACCESS_TYPE;
833 res = buffer->SetMetadata(key, values);
834 CHECK_AND_RETURN_LOG(res == 0, "SetSurfaceBufferHebcAccessType: SetMetadata fail! res=%{public}d", res);
835 }
836
CopyMetaData(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer)837 void ImageEffect::CopyMetaData(sptr<SurfaceBuffer> &inBuffer, sptr<SurfaceBuffer> &outBuffer) {
838 std::vector<uint32_t> keys = {};
839 auto res = inBuffer->ListMetadataKeys(keys);
840 CHECK_AND_RETURN_LOG(res == GSError::GSERROR_OK, "CopyMetaData: ListMetadataKeys fail! res=%{public}d", res);
841 for (uint32_t key: keys) {
842 std::vector<uint8_t> values;
843 res = inBuffer->GetMetadata(key, values);
844 if (res != 0) {
845 EFFECT_LOGE("GetMetadata fail! key = %{public}d res = %{public}d", key, res);
846 continue;
847 }
848 auto isNeedUpdate = !(key == VIDEO_SINK_FILTER_STATUS) || !(values[0] == VIDEO_SINK_FILTER_STATUS);
849 impl_->effectContext_->metaInfoNegotiate_->SetNeedUpdate(isNeedUpdate);
850 res = outBuffer->SetMetadata(key, values);
851 if (res != 0) {
852 EFFECT_LOGE("SetMetadata fail! key = %{public}d res = %{public}d", key, res);
853 continue;
854 }
855 }
856 }
857
OnBufferAvailableToProcess(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,int64_t timestamp,bool isNeedRender)858 bool ImageEffect::OnBufferAvailableToProcess(sptr<SurfaceBuffer> &inBuffer, sptr<SurfaceBuffer> &outBuffer,
859 int64_t timestamp, bool isNeedRender)
860 {
861 bool isNeedSwap = true;
862 if (isNeedRender) {
863 CopyMetaData(inBuffer, outBuffer);
864 inDateInfo_.surfaceBufferInfo_ = {
865 .surfaceBuffer_ = inBuffer,
866 .timestamp_ = timestamp,
867 };
868 outDateInfo_.surfaceBufferInfo_ = {
869 .surfaceBuffer_ = outBuffer,
870 .timestamp_ = timestamp,
871 };
872 ErrorCode res = this->Render();
873 isNeedSwap = (res != ErrorCode::SUCCESS);
874 }
875
876 auto detRet = GSError::GSERROR_OK;
877 if (isNeedSwap) {
878 EFFECT_TRACE_BEGIN("OnBufferAvailableToProcess::SwapBuffers");
879 detRet = toProducerSurface_->DetachBufferFromQueue(outBuffer);
880 CHECK_AND_RETURN_RET_LOG(detRet == GSError::GSERROR_OK, true,
881 "OnBufferAvailableToProcess: detach buffer from producerSurface_ failed");
882 detRet = toProducerSurface_->AttachBufferToQueue(inBuffer);
883 CHECK_AND_RETURN_RET_LOG(detRet == GSError::GSERROR_OK, true,
884 "OnBufferAvailableToProcess: attach buffer from producerSurface_ failed");
885 EFFECT_TRACE_END();
886 }
887 return isNeedSwap;
888 }
889
GetBufferRequestConfig(const sptr<SurfaceBuffer> & buffer)890 BufferRequestConfig ImageEffect::GetBufferRequestConfig(const sptr<SurfaceBuffer>& buffer)
891 {
892 return {
893 .width = buffer->GetWidth(),
894 .height = buffer->GetHeight(),
895 .strideAlignment = 0x8, // default stride is 8 Bytes.
896 .format = buffer->GetFormat(),
897 .usage = buffer->GetUsage(),
898 .timeout = 0,
899 .colorGamut = buffer->GetSurfaceBufferColorGamut(),
900 .transform = buffer->GetSurfaceBufferTransform(),
901 };
902 }
903
FlushBuffer(sptr<SurfaceBuffer> & flushBuffer,int64_t timestamp,bool isNeedRender)904 void ImageEffect::FlushBuffer(sptr<SurfaceBuffer>& flushBuffer, int64_t timestamp, bool isNeedRender) {
905 if (isNeedRender) {
906 EFFECT_TRACE_BEGIN("FlushBuffer::FlushCache");
907 (void)flushBuffer->FlushCache();
908 EFFECT_TRACE_END();
909 }
910
911 BufferFlushConfig flushConfig = {
912 .damage = {
913 .w = flushBuffer->GetWidth(),
914 .h = flushBuffer->GetHeight(),
915 },
916 .timestamp = timestamp,
917 };
918 CHECK_AND_RETURN_LOG(imageEffectFlag_ == STRUCT_IMAGE_EFFECT_CONSTANT,
919 "ImageEffect::OnBufferAvailableWithCPU ImageEffect not exist.");
920 CHECK_AND_RETURN_LOG(toProducerSurface_ != nullptr,
921 "ImageEffect::OnBufferAvailableWithCPU: toProducerSurface is nullptr.");
922 constexpr int32_t invalidFence = -1;
923 toProducerSurface_->FlushBuffer(flushBuffer, invalidFence, flushConfig);
924 }
925
OnBufferAvailableWithCPU(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,const OHOS::Rect & damages,int64_t timestamp)926 bool ImageEffect::OnBufferAvailableWithCPU(sptr<SurfaceBuffer>& inBuffer, sptr<SurfaceBuffer>& outBuffer,
927 const OHOS::Rect& damages, int64_t timestamp)
928 {
929 CHECK_AND_RETURN_RET_LOG(inBuffer != nullptr, true, "ImageEffect::OnBufferAvailableWithCPU: inBuffer is nullptr.");
930 outDateInfo_.dataType_ = DataType::SURFACE;
931 UpdateProducerSurfaceInfo();
932
933 sptr<SyncFence> syncFence = SyncFence::INVALID_FENCE;
934
935 bool isSrcHebcData = IsSurfaceBufferHebc(inBuffer);
936 CHECK_AND_RETURN_RET_LOG(impl_ != nullptr, true, "OnBufferAvailableToProcess: impl is nullptr.");
937 bool isNeedRender = !isSrcHebcData && impl_->effectState_ == EffectState::RUNNING;
938
939 if (isNeedRender) {
940 EFFECT_TRACE_BEGIN("inBuffer::InvalidateCache");
941 (void)inBuffer->InvalidateCache();
942 EFFECT_TRACE_END();
943 }
944
945 auto requestConfig = GetBufferRequestConfig(inBuffer);
946
947 CHECK_AND_RETURN_RET_LOG(toProducerSurface_ != nullptr, true,
948 "OnBufferAvailableWithCPU: toProducerSurface is nullptr.");
949 auto ret = toProducerSurface_->RequestBuffer(outBuffer, syncFence, requestConfig);
950 CHECK_AND_RETURN_RET_LOG(ret == 0 && outBuffer != nullptr, true, "RequestBuffer failed. %{public}d", ret);
951
952 constexpr uint32_t waitForEver = -1;
953 (void)syncFence->Wait(waitForEver);
954
955 EFFECT_LOGD("inBuffer: w=%{public}d h=%{public}d stride=%{public}d len=%{public}d usage=%{public}lld",
956 inBuffer->GetWidth(), inBuffer->GetHeight(), inBuffer->GetStride(), inBuffer->GetSize(),
957 static_cast<unsigned long long>(inBuffer->GetUsage()));
958 EFFECT_LOGD("outBuffer: w=%{public}d h=%{public}d stride=%{public}d len=%{public}d usage=%{public}lld",
959 outBuffer->GetWidth(), outBuffer->GetHeight(), outBuffer->GetStride(), outBuffer->GetSize(),
960 static_cast<unsigned long long>(outBuffer->GetUsage()));
961
962 SetSurfaceBufferHebcAccessType(outBuffer,
963 isSrcHebcData ? V1_1::HebcAccessType::HEBC_ACCESS_HW_ONLY : V1_1::HebcAccessType::HEBC_ACCESS_CPU_ACCESS);
964 bool isNeedSwap = OnBufferAvailableToProcess(inBuffer, outBuffer, timestamp, isNeedRender);
965
966 auto flushBuffer = (isNeedSwap ? inBuffer : outBuffer);
967 FlushBuffer(flushBuffer, timestamp, isNeedRender);
968
969 return isNeedSwap;
970 }
971
ConsumerBufferAvailable(sptr<SurfaceBuffer> & inBuffer,sptr<SurfaceBuffer> & outBuffer,const OHOS::Rect & damages,int64_t timestamp)972 bool ImageEffect::ConsumerBufferAvailable(sptr<SurfaceBuffer>& inBuffer, sptr<SurfaceBuffer>& outBuffer,
973 const OHOS::Rect& damages, int64_t timestamp)
974 {
975 CHECK_AND_RETURN_RET_LOG(imageEffectFlag_ == STRUCT_IMAGE_EFFECT_CONSTANT, true,
976 "ImageEffect::OnBufferAvailable ImageEffect not exist.");
977 std::unique_lock<std::mutex> lock(innerEffectMutex);
978 return OnBufferAvailableWithCPU(inBuffer, outBuffer, damages, timestamp);
979 }
980
GetInputSurface()981 sptr<Surface> ImageEffect::GetInputSurface()
982 {
983 inDateInfo_.dataType_ = DataType::SURFACE;
984 if (fromProducerSurface_ != nullptr) {
985 return fromProducerSurface_;
986 }
987
988 if (impl_->surfaceAdapter_ == nullptr) {
989 impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
990 }
991
992 if (impl_->surfaceAdapter_) {
993 fromProducerSurface_ = impl_->surfaceAdapter_->GetProducerSurface();
994 }
995
996 auto consumerListener = [this](sptr<SurfaceBuffer>& inBuffer,
997 sptr<SurfaceBuffer>& outBuffer, const OHOS::Rect& damages, int64_t timestamp) {
998 return ConsumerBufferAvailable(inBuffer, outBuffer, damages, timestamp);
999 };
1000
1001 if (impl_->surfaceAdapter_) {
1002 impl_->surfaceAdapter_->SetConsumerListener(std::move(consumerListener));
1003 }
1004
1005 return fromProducerSurface_;
1006 }
1007
SetOutNativeWindow(OHNativeWindow * nativeWindow)1008 ErrorCode ImageEffect::SetOutNativeWindow(OHNativeWindow *nativeWindow)
1009 {
1010 CHECK_AND_RETURN_RET_LOG(nativeWindow != nullptr, ErrorCode::ERR_INPUT_NULL, "nativeWindow is nullptr");
1011 OHOS::sptr<OHOS::Surface> surface = nativeWindow->surface;
1012 CHECK_AND_RETURN_RET_LOG(surface != nullptr, ErrorCode::ERR_INPUT_NULL, "surface is nullptr");
1013 toProducerSurface_ = surface;
1014 outDateInfo_.dataType_ = DataType::NATIVE_WINDOW;
1015 impl_->effectContext_->renderEnvironment_->InitEngine(nativeWindow);
1016 if (impl_->surfaceAdapter_ == nullptr) {
1017 impl_->surfaceAdapter_ = std::make_unique<EffectSurfaceAdapter>();
1018 }
1019 impl_->surfaceAdapter_->SetOutputSurfaceDefaultUsage(toProducerSurface_->GetDefaultUsage());
1020
1021 toProducerSurface_->SetTransform(GRAPHIC_ROTATE_BUTT);
1022 return ErrorCode::SUCCESS;
1023 }
1024
Configure(const std::string & key,const Plugin::Any & value)1025 ErrorCode ImageEffect::Configure(const std::string &key, const Plugin::Any &value)
1026 {
1027 auto configTypeIt = std::find_if(configTypeTab_.begin(), configTypeTab_.end(),
1028 [&key](const std::pair<std::string, ConfigType> &item) { return item.first.compare(key) == 0; });
1029
1030 CHECK_AND_RETURN_RET_LOG(configTypeIt != configTypeTab_.end(), ErrorCode::ERR_UNSUPPORTED_CONFIG_TYPE,
1031 "config key is not support! key=%{public}s", key.c_str());
1032
1033 ConfigType configType = configTypeIt->second;
1034 switch (configType) {
1035 case ConfigType::IPTYPE: {
1036 int32_t runningType;
1037 ErrorCode result = CommonUtils::ParseAny(value, runningType);
1038 CHECK_AND_RETURN_RET_LOG(result == ErrorCode::SUCCESS, result,
1039 "parse any fail! expect type is uint32_t! key=%{public}s", key.c_str());
1040 auto it = std::find_if(runningTypeTab_.begin(), runningTypeTab_.end(),
1041 [&runningType](const std::pair<int32_t, std::vector<IPType>> &item) {
1042 return item.first == runningType;
1043 });
1044 CHECK_AND_RETURN_RET_LOG(it != runningTypeTab_.end(), ErrorCode::ERR_UNSUPPORTED_RUNNINGTYPE,
1045 "not support runningType! key=%{public}s, runningType=%{public}d", key.c_str(), runningType);
1046 config_[configType] = it->second;
1047 break;
1048 }
1049 default:
1050 EFFECT_LOGE("config type is not support! configType=%{public}d", configType);
1051 return ErrorCode::ERR_UNSUPPORTED_CONFIG_TYPE;
1052 }
1053 return ErrorCode::SUCCESS;
1054 }
1055
ClearDataInfo(DataInfo & dataInfo)1056 void ImageEffect::ClearDataInfo(DataInfo &dataInfo)
1057 {
1058 dataInfo.dataType_ = DataType::UNKNOWN;
1059 dataInfo.pixelMap_ = nullptr;
1060 dataInfo.surfaceBufferInfo_.surfaceBuffer_ = nullptr;
1061 dataInfo.surfaceBufferInfo_.timestamp_ = 0;
1062 dataInfo.uri_ = "";
1063 dataInfo.path_ = "";
1064 }
1065
IsSameInOutputData(const DataInfo & inDataInfo,const DataInfo & outDataInfo)1066 bool IsSameInOutputData(const DataInfo &inDataInfo, const DataInfo &outDataInfo)
1067 {
1068 if (inDataInfo.dataType_ != outDataInfo.dataType_) {
1069 return false;
1070 }
1071
1072 switch (inDataInfo.dataType_) {
1073 case DataType::PIXEL_MAP:
1074 return inDataInfo.pixelMap_ == outDataInfo.pixelMap_;
1075 case DataType::SURFACE_BUFFER:
1076 return inDataInfo.surfaceBufferInfo_.surfaceBuffer_ == outDataInfo.surfaceBufferInfo_.surfaceBuffer_;
1077 case DataType::PATH:
1078 return inDataInfo.path_ == outDataInfo.path_;
1079 case DataType::URI:
1080 return inDataInfo.uri_ == outDataInfo.uri_;
1081 case DataType::PICTURE:
1082 return inDataInfo.picture_ == outDataInfo.picture_;
1083 default:
1084 return false;
1085 }
1086 }
1087
LockAll(std::shared_ptr<EffectBuffer> & srcEffectBuffer,std::shared_ptr<EffectBuffer> & dstEffectBuffer)1088 ErrorCode ImageEffect::LockAll(std::shared_ptr<EffectBuffer> &srcEffectBuffer,
1089 std::shared_ptr<EffectBuffer> &dstEffectBuffer)
1090 {
1091 ErrorCode res = ParseDataInfo(inDateInfo_, srcEffectBuffer, false);
1092 if (res != ErrorCode::SUCCESS) {
1093 EFFECT_LOGE("ParseDataInfo inData fail! res=%{public}d", res);
1094 return res;
1095 }
1096 EFFECT_LOGI("input data set, parse data info success! dataType=%{public}d", inDateInfo_.dataType_);
1097
1098 if (outDateInfo_.dataType_ != DataType::UNKNOWN && !IsSameInOutputData(inDateInfo_, outDateInfo_)) {
1099 EFFECT_LOGI("output data set, start parse data info. dataType=%{public}d", outDateInfo_.dataType_);
1100 res = ParseDataInfo(outDateInfo_, dstEffectBuffer, true);
1101 if (res != ErrorCode::SUCCESS) {
1102 EFFECT_LOGE("ParseDataInfo outData fail! res=%{public}d", res);
1103 return res;
1104 }
1105 EFFECT_LOGI("output data set, parse data info success! dataType=%{public}d", outDateInfo_.dataType_);
1106 }
1107
1108 return ErrorCode::SUCCESS;
1109 }
1110
ParseDataInfo(DataInfo & dataInfo,std::shared_ptr<EffectBuffer> & effectBuffer,bool isOutputData)1111 ErrorCode ImageEffect::ParseDataInfo(DataInfo &dataInfo, std::shared_ptr<EffectBuffer> &effectBuffer,
1112 bool isOutputData)
1113 {
1114 switch (dataInfo.dataType_) {
1115 case DataType::PIXEL_MAP:
1116 return CommonUtils::LockPixelMap(dataInfo.pixelMap_, effectBuffer);
1117 case DataType::SURFACE:
1118 case DataType::SURFACE_BUFFER:
1119 return CommonUtils::ParseSurfaceData(dataInfo.surfaceBufferInfo_.surfaceBuffer_, effectBuffer,
1120 dataInfo.dataType_, dataInfo.surfaceBufferInfo_.timestamp_);
1121 case DataType::URI:
1122 return CommonUtils::ParseUri(dataInfo.uri_, effectBuffer, isOutputData);
1123 case DataType::PATH:
1124 return CommonUtils::ParsePath(dataInfo.path_, effectBuffer, isOutputData);
1125 case DataType::NATIVE_WINDOW:
1126 return CommonUtils::ParseNativeWindowData(effectBuffer, dataInfo.dataType_);
1127 case DataType::PICTURE:
1128 return CommonUtils::ParsePicture(dataInfo.picture_, effectBuffer);
1129 case DataType::UNKNOWN:
1130 EFFECT_LOGW("dataType is unknown! Data is not set!");
1131 return ErrorCode::ERR_NO_DATA;
1132 default:
1133 EFFECT_LOGW("dataType is not support! dataType=%{public}d", dataInfo.dataType_);
1134 return ErrorCode::ERR_UNSUPPORTED_DATA_TYPE;
1135 }
1136 }
1137
UnLockAll()1138 void ImageEffect::UnLockAll()
1139 {
1140 UnLockData(inDateInfo_);
1141 UnLockData(outDateInfo_);
1142 }
1143
UnLockData(DataInfo & dataInfo)1144 void ImageEffect::UnLockData(DataInfo &dataInfo)
1145 {
1146 switch (dataInfo.dataType_) {
1147 case DataType::PIXEL_MAP: {
1148 CommonUtils::UnlockPixelMap(dataInfo.pixelMap_);
1149 return;
1150 }
1151 default:
1152 return;
1153 }
1154 }
1155
InitEGLEnv()1156 void ImageEffect::InitEGLEnv()
1157 {
1158 EFFECT_TRACE_NAME("ImageEffect::InitEGLEnv");
1159 impl_->effectContext_->renderEnvironment_->Init();
1160 impl_->effectContext_->renderEnvironment_->Prepare();
1161 }
1162
DestroyEGLEnv()1163 void ImageEffect::DestroyEGLEnv()
1164 {
1165 EFFECT_LOGI("ImageEffect DestroyEGLEnv enter!");
1166 if (impl_->effectContext_->renderEnvironment_ == nullptr) {
1167 return;
1168 }
1169 impl_->effectContext_->renderEnvironment_->ReleaseParam();
1170 impl_->effectContext_->renderEnvironment_->Release();
1171 EFFECT_LOGI("ImageEffect DestroyEGLEnv end!");
1172 }
1173
SetExtraInfo(EffectJsonPtr res)1174 ErrorCode ImageEffect::SetExtraInfo(EffectJsonPtr res)
1175 {
1176 extraInfo_ = res;
1177 return ErrorCode::SUCCESS;
1178 }
1179
ExtInitModule()1180 void ImageEffect::ExtInitModule()
1181 {
1182 }
1183
ExtDeinitModule()1184 void ImageEffect::ExtDeinitModule()
1185 {
1186 }
1187
SetInputPicture(Picture * picture)1188 ErrorCode ImageEffect::SetInputPicture(Picture *picture)
1189 {
1190 EFFECT_LOGD("ImageEffect::SetInputPicture");
1191 CHECK_AND_RETURN_RET_LOG(picture != nullptr, ErrorCode::ERR_INPUT_NULL,
1192 "ImageEffect::SetInputPicture: picture is null!");
1193
1194 impl_->effectContext_->renderEnvironment_->NotifyInputChanged();
1195 ClearDataInfo(inDateInfo_);
1196 inDateInfo_.dataType_ = DataType::PICTURE;
1197 inDateInfo_.picture_ = picture;
1198
1199 return ErrorCode::SUCCESS;
1200 }
1201
SetOutputPicture(Picture * picture)1202 ErrorCode ImageEffect::SetOutputPicture(Picture *picture)
1203 {
1204 EFFECT_LOGD("ImageEffect::SetOutputPicture");
1205 ClearDataInfo(outDateInfo_);
1206 if (picture == nullptr) {
1207 EFFECT_LOGI("SetOutputPicture:picture set to null!");
1208 return ErrorCode::SUCCESS;
1209 }
1210 outDateInfo_.dataType_ = DataType::PICTURE;
1211 outDateInfo_.picture_ = picture;
1212
1213 return ErrorCode::SUCCESS;
1214 }
1215 } // namespace Effect
1216 } // namespace Media
1217 } // namespace OHOS