1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "scene_adapter/scene_adapter.h"
17
18 #include <dlfcn.h>
19 #include <memory>
20 #include <string_view>
21
22 #include "napi/native_api.h"
23 #include "napi/native_node_api.h"
24
25 #include <base/containers/array_view.h>
26 #include <base/containers/shared_ptr.h>
27
28 #include <core/intf_engine.h>
29 #include <core/ecs/intf_system_graph_loader.h>
30 #include <core/engine_info.h>
31 #include <core/image/intf_image_loader_manager.h>
32 #include <core/implementation_uids.h>
33 #include <core/io/intf_file_manager.h>
34 #include <core/namespace.h>
35 #include <core/os/intf_platform.h>
36 #include <core/plugin/intf_plugin_register.h>
37 #include <core/property/intf_property_handle.h>
38
39 #include <jpg/implementation_uids.h>
40
41 #include <meta/interface/intf_meta_object_lib.h>
42 #include <meta/interface/intf_task_queue_registry.h>
43 #include <meta/interface/intf_task_queue.h>
44 #include <meta/interface/intf_object.h>
45 #include <meta/interface/intf_object_registry.h>
46 #include <meta/interface/intf_task_queue.h>
47 #include <meta/base/interface_macros.h>
48 #include <meta/api/make_callback.h>
49 #include <meta/ext/object.h>
50
51 #include <png/implementation_uids.h>
52 #include <scene/base/namespace.h>
53 #include <scene/interface/intf_scene.h>
54 #include <3d/ecs/components/environment_component.h>
55 #include <3d/ecs/components/render_handle_component.h>
56 #include <3d/ecs/components/uri_component.h>
57 #include <scene/ext/intf_internal_scene.h>
58 #include <scene/interface/intf_mesh.h>
59 #include <scene/interface/intf_material.h>
60 #include <scene/ext/intf_render_resource.h>
61 #include <scene/interface/intf_camera.h>
62 #include <scene/ext/intf_internal_scene.h>
63 #include <scene/ext/intf_ecs_context.h>
64 #include <scene/interface/intf_application_context.h>
65
66 #include <scene/ext/intf_ecs_object_access.h>
67 #include <text_3d/implementation_uids.h>
68 #include "ability.h"
69
70 #include "data_ability_helper.h"
71 #include <sys/resource.h>
72 #include "napi_base_context.h"
73
74 #include <render/implementation_uids.h>
75 #include <render/gles/intf_device_gles.h>
76 #include <render/intf_renderer.h>
77 #include <render/intf_render_context.h>
78 #include <render/util/intf_render_util.h>
79 #include <render/vulkan/intf_device_vk.h>
80
81 #include "3d_widget_adapter_log.h"
82 #include "widget_trace.h"
83 #include "ohos/texture_layer.h"
84 #include "lume_render_config.h"
85
86 #include <cam_preview/namespace.h>
87 #include <cam_preview/implementation_uids.h>
88
89 // surface buffer
90 #include <native_buffer.h>
91 #include <surface_buffer.h>
92 #include <securec.h>
93
94 #include <EGL/egl.h>
95 #include <EGL/eglext.h>
96 #include <GLES/gl.h>
97 #include <GLES/glext.h>
98 #define ENGINE_SERVICE_PRIORITY (-20)
99 namespace OHOS::Render3D {
100 #define RETURN_IF_NULL(ptr) \
101 do { \
102 if (!(ptr)) { \
103 WIDGET_LOGE("%s is null in %s", #ptr, __func__); \
104 return; \
105 } \
106 } while (0)
107
108 #define RETURN_FALSE_IF_NULL(ptr) \
109 do { \
110 if (!(ptr)) { \
111 WIDGET_LOGE("%s is null in %s", #ptr, __func__); \
112 return false; \
113 } \
114 } while (0)
115
GetHapInfo()116 HapInfo GetHapInfo()
117 {
118 WIDGET_SCOPED_TRACE("SceneAdapter::GetHapInfo");
119 std::shared_ptr<AbilityRuntime::ApplicationContext> context =
120 AbilityRuntime::ApplicationContext::GetApplicationContext();
121 if (!context) {
122 WIDGET_LOGE("Failed to get application context.");
123 return {};
124 }
125 auto resourceManager = context->GetResourceManager();
126 if (!resourceManager) {
127 WIDGET_LOGE("Failed to get resource manager.");
128 return {};
129 }
130 HapInfo hapInfo;
131 hapInfo.bundleName_ = resourceManager->bundleInfo.first;
132 hapInfo.moduleName_ = resourceManager->bundleInfo.second;
133 hapInfo.resourceManager_ = context->CreateModuleResourceManager(hapInfo.bundleName_, hapInfo.moduleName_);
134 // hapPath
135 std::string hapPath = context->GetBundleCodeDir();
136 hapInfo.hapPath_ = hapPath + "/" + hapInfo.moduleName_ + ".hap";
137 WIDGET_LOGD("bundle %s, module %s, hapPath %s",
138 hapInfo.bundleName_.c_str(),
139 hapInfo.moduleName_.c_str(),
140 hapInfo.hapPath_.c_str());
141
142 return hapInfo;
143 }
144
145 using IntfPtr = META_NS::SharedPtrIInterface;
146 using IntfWeakPtr = META_NS::WeakPtrIInterface;
147
148 struct EngineInstance {
149 void *libHandle_ = nullptr;
150 SCENE_NS::IApplicationContext::Ptr applicationContext_;
151 BASE_NS::shared_ptr<RENDER_NS::IRenderContext> renderContext_;
152 BASE_NS::shared_ptr<CORE_NS::IEngine> engine_;
153 };
154
155 static EngineInstance engineInstance_;
156 static std::mutex mute;
157 static HapInfo hapInfo_;
158 META_NS::ITaskQueue::Ptr engineThread;
159 META_NS::ITaskQueue::Ptr ioThread;
160 META_NS::ITaskQueue::Ptr releaseThread;
161 META_NS::ITaskQueue::Token renderTask {};
162
LockCompositor()163 void LockCompositor()
164 {
165 mute.lock();
166 }
167
UnlockCompositor()168 void UnlockCompositor()
169 {
170 mute.unlock();
171 }
172
173 static constexpr BASE_NS::Uid ENGINE_THREAD { "2070e705-d061-40e4-bfb7-90fad2c280af" };
174 static constexpr BASE_NS::Uid APP_THREAD { "b2e8cef3-453a-4651-b564-5190f8b5190d" };
175 static constexpr BASE_NS::Uid IO_QUEUE { "be88e9a0-9cd8-45ab-be48-937953dc258f" };
176 static constexpr BASE_NS::Uid JS_RELEASE_THREAD { "3784fa96-b25b-4e9c-bbf1-e897d36f73af" };
177 static constexpr uint32_t FLOAT_TO_BYTE = sizeof(float) / sizeof(uint8_t);
178 static constexpr uint32_t TRANSFORM_MATRIX_SIZE = 16;
179 static const char* const RENDER_DATA_STORE_DEFAULT_STAGING = "RenderDataStoreDefaultStaging";
180 static const char* const ENV_PROPERTIES_BUFFER_NAME = "AR_CAMERA_TRANSFORM_MATRIX";
181 static constexpr uint64_t BUFFER_FENCE_HANDLE_BASE { 0xAAAAAAAAaaaaaaaa };
182 static constexpr uint32_t TIME_OUT_MILLISECONDS = 5000;
183
184 template<typename T>
LoadFunc(T & fn,const char * fName,void * handle)185 bool LoadFunc(T &fn, const char *fName, void* handle)
186 {
187 fn = reinterpret_cast<T>(dlsym(handle, fName));
188 if (fn == nullptr) {
189 WIDGET_LOGE("%s open %s", __func__, dlerror());
190 return false;
191 }
192 return true;
193 }
194
~SceneAdapter()195 SceneAdapter::~SceneAdapter()
196 {
197 }
198
SceneAdapter()199 SceneAdapter::SceneAdapter()
200 {
201 WIDGET_LOGD("scene adapter Impl create");
202 }
203
LoadEngineLib()204 bool SceneAdapter::LoadEngineLib()
205 {
206 if (engineInstance_.libHandle_ != nullptr) {
207 WIDGET_LOGD("%s, already loaded", __func__);
208 return true;
209 }
210
211 #define TO_STRING(name) #name
212 #define LIB_NAME(name) TO_STRING(name)
213 constexpr std::string_view lib { LIB_NAME(LIB_ENGINE_CORE)".so" };
214 engineInstance_.libHandle_ = dlopen(lib.data(), RTLD_LAZY);
215
216 if (engineInstance_.libHandle_ == nullptr) {
217 WIDGET_LOGE("%s, open lib fail %s", __func__, dlerror());
218 }
219 #undef TO_STRING
220 #undef LIB_NAME
221
222 #define LOAD_FUNC(fn, name) LoadFunc<decltype(fn)>(fn, name, engineInstance_.libHandle_)
223 if (!(LOAD_FUNC(CORE_NS::CreatePluginRegistry,
224 "_ZN4Core20CreatePluginRegistryERKNS_18PlatformCreateInfoE")
225 && LOAD_FUNC(CORE_NS::GetPluginRegister, "_ZN4Core17GetPluginRegisterEv")
226 && LOAD_FUNC(CORE_NS::IsDebugBuild, "_ZN4Core12IsDebugBuildEv")
227 && LOAD_FUNC(CORE_NS::GetVersion, "_ZN4Core13GetVersionRevEv"))) {
228 return false;
229 }
230 #undef LOAD_FUNC
231
232 return true;
233 }
234
LoadPlugins(const CORE_NS::PlatformCreateInfo & platformCreateInfo)235 bool SceneAdapter::LoadPlugins(const CORE_NS::PlatformCreateInfo& platformCreateInfo)
236 {
237 if (engineInstance_.libHandle_) {
238 WIDGET_LOGI("%s, already loaded", __func__);
239 return true;
240 }
241 if (!LoadEngineLib()) {
242 return false;
243 }
244 WIDGET_LOGD("load engine suceess!");
245
246 BASE_NS::vector<BASE_NS::Uid> DefaultPluginVector = {
247 CAM_PREVIEW_NS::UID_CAMERA_PREVIEW_PLUGIN,
248 SCENE_NS::UID_SCENE_PLUGIN,
249 #if defined(USE_LIB_PNG_JPEG_DYNAMIC_PLUGIN) && USE_LIB_PNG_JPEG_DYNAMIC_PLUGIN
250 JPGPlugin::UID_JPG_PLUGIN,
251 PNGPlugin::UID_PNG_PLUGIN,
252 #endif
253 };
254 const BASE_NS::array_view<BASE_NS::Uid> DefaultPluginList(DefaultPluginVector.data(), DefaultPluginVector.size());
255 CORE_NS::CreatePluginRegistry(platformCreateInfo);
256 if (!CORE_NS::GetPluginRegister().LoadPlugins(DefaultPluginList)) {
257 WIDGET_LOGE("fail to load scene widget plugin");
258 return false;
259 }
260 WIDGET_LOGI("load plugin success");
261 return true;
262 }
263
InitEngine(CORE_NS::PlatformCreateInfo platformCreateInfo)264 bool SceneAdapter::InitEngine(CORE_NS::PlatformCreateInfo platformCreateInfo)
265 {
266 auto& tr = META_NS::GetTaskQueueRegistry();
267 auto& obr = META_NS::GetObjectRegistry();
268
269 engineThread = tr.GetTaskQueue(ENGINE_THREAD);
270 if (!engineThread) {
271 engineThread = obr.Create<META_NS::ITaskQueue>(META_NS::ClassId::ThreadedTaskQueue);
272 tr.RegisterTaskQueue(engineThread, ENGINE_THREAD);
273 }
274 ioThread = tr.GetTaskQueue(IO_QUEUE);
275 if (!ioThread) {
276 ioThread = obr.Create<META_NS::ITaskQueue>(META_NS::ClassId::ThreadedTaskQueue);
277 tr.RegisterTaskQueue(ioThread, IO_QUEUE);
278 }
279 releaseThread = tr.GetTaskQueue(JS_RELEASE_THREAD);
280 if (!releaseThread) {
281 auto &obr = META_NS::GetObjectRegistry();
282 releaseThread = obr.Create<META_NS::ITaskQueue>(META_NS::ClassId::ThreadedTaskQueue);
283 tr.RegisterTaskQueue(releaseThread, JS_RELEASE_THREAD);
284 }
285
286 bool inited = false;
287 auto initCheck = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([&inited]() {
288 inited = (engineInstance_.engine_ != nullptr);
289 return META_NS::IAny::Ptr {};
290 });
291 engineThread->AddWaitableTask(initCheck)->Wait();
292 if (inited) {
293 WIDGET_LOGI("engine already inited");
294 return true;
295 }
296
297 auto engineInit = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([platformCreateInfo]() {
298 setpriority(0, 0, ENGINE_SERVICE_PRIORITY);
299 auto &obr = META_NS::GetObjectRegistry();
300 // Initialize lumeengine/render etc
301 CORE_NS::EngineCreateInfo engineCreateInfo{platformCreateInfo, {}, {}};
302 if (auto factory = CORE_NS::GetInstance<CORE_NS::IEngineFactory>(CORE_NS::UID_ENGINE_FACTORY)) {
303 engineInstance_.engine_.reset(factory->Create(engineCreateInfo).get());
304 } else {
305 WIDGET_LOGE("could not get engine factory");
306 return META_NS::IAny::Ptr {};
307 }
308 if (!engineInstance_.engine_) {
309 WIDGET_LOGE("get engine fail");
310 return META_NS::IAny::Ptr {};
311 }
312 auto &fileManager = engineInstance_.engine_->GetFileManager();
313 const auto &platform = engineInstance_.engine_->GetPlatform();
314 platform.RegisterDefaultPaths(fileManager);
315 engineInstance_.engine_->Init();
316
317 RENDER_NS::RenderCreateInfo renderCreateInfo{};
318 RENDER_NS::BackendExtraGLES glExtra{};
319 Render::DeviceCreateInfo deviceCreateInfo{};
320
321 std::string backendProp = LumeRenderConfig::GetInstance().renderBackend_ == "force_vulkan" ? "vulkan" : "gles";
322 if (backendProp == "vulkan") {
323 WIDGET_LOGI("backend vulkan");
324 } else {
325 WIDGET_LOGI("backend gles");
326 glExtra.depthBits = 24; // 24 : bits size
327 glExtra.sharedContext = EGL_NO_CONTEXT;
328 deviceCreateInfo.backendType = RENDER_NS::DeviceBackendType::OPENGLES;
329 deviceCreateInfo.backendConfiguration = &glExtra;
330 renderCreateInfo.applicationInfo = {};
331 renderCreateInfo.deviceCreateInfo = deviceCreateInfo;
332 }
333
334 engineInstance_.renderContext_.reset(
335 static_cast<RENDER_NS::IRenderContext *>(
336 engineInstance_.engine_->GetInterface<CORE_NS::IClassFactory>()
337 ->CreateInstance(RENDER_NS::UID_RENDER_CONTEXT)
338 .release()));
339
340 auto rrc = engineInstance_.renderContext_->Init(renderCreateInfo);
341 if (rrc != RENDER_NS::RenderResultCode::RENDER_SUCCESS) {
342 WIDGET_LOGE("Failed to create render context");
343 return META_NS::IAny::Ptr {};
344 }
345
346 // Save the stuff to the default object context.
347 auto engineThread = META_NS::GetTaskQueueRegistry().GetTaskQueue(ENGINE_THREAD);
348 // thats the javascript thread...
349 auto appThread = engineThread;
350 auto doc = interface_cast<META_NS::IMetadata>(obr.GetDefaultObjectContext());
351 if (doc == nullptr) {
352 WIDGET_LOGE("nullptr from interface_cast");
353 return META_NS::IAny::Ptr {};
354 }
355 auto flags = META_NS::ObjectFlagBits::INTERNAL | META_NS::ObjectFlagBits::NATIVE;
356
357 doc->AddProperty(META_NS::ConstructProperty<IntfPtr>("EngineQueue", nullptr, flags));
358 doc->AddProperty(META_NS::ConstructProperty<IntfPtr>("AppQueue", nullptr, flags));
359 doc->AddProperty(META_NS::ConstructArrayProperty<IntfWeakPtr>("Scenes", {}, flags));
360
361 // Use engine file manager for our resource manager
362 auto resources =
363 META_NS::GetObjectRegistry().Create<CORE_NS::IResourceManager>(
364 META_NS::ClassId::FileResourceManager);
365 resources->SetFileManager(CORE_NS::IFileManager::Ptr(&fileManager));
366
367 engineInstance_.applicationContext_ =
368 META_NS::GetObjectRegistry().Create<SCENE_NS::IApplicationContext>(
369 SCENE_NS::ClassId::ApplicationContext);
370 if (engineInstance_.applicationContext_) {
371 SCENE_NS::IApplicationContext::ApplicationContextInfo info{
372 engineThread, appThread, engineInstance_.renderContext_,
373 resources, SCENE_NS::SceneOptions{}};
374 engineInstance_.applicationContext_->Initialize(info);
375 }
376
377 doc->GetProperty<META_NS::SharedPtrIInterface>("EngineQueue")->SetValue(engineThread);
378 doc->GetProperty<META_NS::SharedPtrIInterface>("AppQueue")->SetValue(appThread);
379 doc->AddProperty(META_NS::ConstructProperty<SCENE_NS::IRenderContext::Ptr>(
380 "RenderContext", engineInstance_.applicationContext_->GetRenderContext(), flags));
381
382 WIDGET_LOGD("register shader paths");
383 static constexpr const RENDER_NS::IShaderManager::ShaderFilePathDesc desc { "shaders://" };
384 engineInstance_.engine_->GetFileManager().RegisterPath("shaders", "OhosRawFile://shaders", false);
385 engineInstance_.renderContext_->GetDevice().GetShaderManager().LoadShaderFiles(desc);
386
387 engineInstance_.engine_->GetFileManager().RegisterPath("appshaders", "OhosRawFile://shaders", false);
388 engineInstance_.engine_->GetFileManager().RegisterPath("apppipelinelayouts",
389 "OhosRawFile:///pipelinelayouts/", true);
390 engineInstance_.engine_->GetFileManager().RegisterPath("fonts", "OhosRawFile:///fonts", true);
391
392 static constexpr const RENDER_NS::IShaderManager::ShaderFilePathDesc desc1 { "appshaders://" };
393 engineInstance_.renderContext_->GetDevice().GetShaderManager().LoadShaderFiles(desc1);
394
395 // "render" one frame. this is to initialize all the default resources etc.
396 // if we never render a single frame, we will get "false positive" leaks of gpu resources.
397 engineInstance_.renderContext_->GetRenderer().RenderFrame({});
398 WIDGET_LOGI("init engine success");
399 return META_NS::IAny::Ptr {};
400 });
401 engineThread->AddWaitableTask(engineInit)->Wait();
402 return true;
403 }
404
SetSceneObj(META_NS::IObject::Ptr pt)405 void SceneAdapter::SetSceneObj(META_NS::IObject::Ptr pt)
406 {
407 WIDGET_LOGD("SceneAdapterImpl::SetSceneObj");
408 sceneWidgetObj_ = pt;
409 }
410
CreateTextureLayer()411 std::shared_ptr<TextureLayer> SceneAdapter::CreateTextureLayer()
412 {
413 WIDGET_LOGI("SceneAdapter::CreateTextureLayer: %u", key_);
414 auto cb = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([this]() {
415 textureLayer_ = std::make_shared<TextureLayer>(key_);
416 key_++;
417 return META_NS::IAny::Ptr {};
418 });
419 if (engineThread) {
420 engineThread->AddWaitableTask(cb)->Wait();
421 } else {
422 WIDGET_LOGE("ENGINE_THREAD not ready in CreateTextureLayer");
423 }
424 return textureLayer_;
425 }
426
LoadPluginsAndInit()427 bool SceneAdapter::LoadPluginsAndInit()
428 {
429 LockCompositor(); // an APP_FREEZE here, so add lock just in case, but suspect others' error
430 WIDGET_LOGI("scene adapter loadPlugins");
431
432 if (hapInfo_.hapPath_ == "") {
433 hapInfo_ = GetHapInfo();
434 }
435
436 #define TO_STRING(name) #name
437 #define PLATFORM_PATH_NAME(name) TO_STRING(name)
438 CORE_NS::PlatformCreateInfo platformCreateInfo {
439 PLATFORM_PATH_NAME(PLATFORM_CORE_ROOT_PATH),
440 PLATFORM_PATH_NAME(PLATFORM_CORE_PLUGIN_PATH),
441 PLATFORM_PATH_NAME(PLATFORM_APP_ROOT_PATH),
442 PLATFORM_PATH_NAME(PLATFORM_APP_PLUGIN_PATH),
443 hapInfo_.hapPath_.c_str(),
444 hapInfo_.bundleName_.c_str(),
445 hapInfo_.moduleName_.c_str(),
446 hapInfo_.resourceManager_
447 };
448 #undef TO_STRING
449 #undef PLATFORM_PATH_NAME
450 if (!LoadPlugins(platformCreateInfo)) {
451 UnlockCompositor();
452 return false;
453 }
454
455 if (!InitEngine(platformCreateInfo)) {
456 UnlockCompositor();
457 return false;
458 }
459
460 CreateRenderFunction();
461 UnlockCompositor();
462 return true;
463 }
464
OnWindowChange(const WindowChangeInfo & windowChangeInfo)465 void SceneAdapter::OnWindowChange(const WindowChangeInfo &windowChangeInfo)
466 {
467 WIDGET_LOGI("SceneAdapter::OnWindowchange");
468 auto cb = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([this, &windowChangeInfo]() {
469 if (!textureLayer_) {
470 WIDGET_LOGE("textureLayer is null in OnWindowChange");
471 return META_NS::IAny::Ptr {};
472 }
473 textureLayer_->OnWindowChange(windowChangeInfo);
474 const auto& textureInfo = textureLayer_->GetTextureInfo();
475 auto& device = engineInstance_.renderContext_->GetDevice();
476 RENDER_NS::SwapchainCreateInfo swapchainCreateInfo {
477 0U,
478 RENDER_NS::SwapchainFlagBits::CORE_SWAPCHAIN_COLOR_BUFFER_BIT |
479 RENDER_NS::SwapchainFlagBits::CORE_SWAPCHAIN_DEPTH_BUFFER_BIT |
480 RENDER_NS::SwapchainFlagBits::CORE_SWAPCHAIN_SRGB_BIT,
481 RENDER_NS::ImageUsageFlagBits::CORE_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
482 {
483 reinterpret_cast<uintptr_t>(textureInfo.nativeWindow_),
484 {}, // instance is not needed for eglCreateSurface or vkCreateSurfaceOHOS
485 }
486 };
487 swapchainHandle_ = device.CreateSwapchainHandle(swapchainCreateInfo, swapchainHandle_, {});
488
489 auto &obr = META_NS::GetObjectRegistry();
490 auto doc = interface_pointer_cast<META_NS::IMetadata>(obr.GetDefaultObjectContext());
491 bitmap_ = obr.Create<SCENE_NS::IRenderTarget>(SCENE_NS::ClassId::Bitmap, doc);
492 if (auto i = interface_cast<SCENE_NS::IRenderResource>(bitmap_)) {
493 i->SetRenderHandle(swapchainHandle_);
494 }
495
496 if (auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_)) {
497 auto cams = scene->GetCameras().GetResult();
498 if (!cams.empty()) {
499 for (auto c : cams) {
500 AttachSwapchain(interface_pointer_cast<META_NS::IObject>(c));
501 }
502 }
503 }
504 return META_NS::IAny::Ptr {};
505 });
506
507 if (engineThread) {
508 engineThread->AddWaitableTask(cb)->Wait();
509 onWindowChanged_ = true;
510 } else {
511 WIDGET_LOGE("ENGINE_THREAD not ready in OnWindowChange");
512 }
513 }
514
CreateRenderFunction()515 void SceneAdapter::CreateRenderFunction()
516 {
517 propSyncSync_ = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([this]() {
518 PropSync();
519 return META_NS::IAny::Ptr {};
520 });
521 // Task used for oneshot renders
522 singleFrameAsync_ = META_NS::MakeCallback<META_NS::ITaskQueueTask>([this]() {
523 RenderFunction();
524 return 0;
525 });
526 // Task used for oneshot synchronous renders
527 singleFrameSync_ = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([this]() {
528 RenderFunction();
529 return META_NS::IAny::Ptr {};
530 });
531 }
532
DeinitRenderThread()533 void SceneAdapter::DeinitRenderThread()
534 {
535 RETURN_IF_NULL(engineThread);
536 if (renderTask) {
537 engineThread->CancelTask(renderTask);
538 renderTask = nullptr;
539 }
540 auto engine_deinit = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([]() {
541 // destroy all swapchains
542 auto &obr = META_NS::GetObjectRegistry();
543 auto doc = interface_cast<META_NS::IMetadata>(obr.GetDefaultObjectContext());
544 if (doc == nullptr) {
545 WIDGET_LOGE("nullptr from interface_cast");
546 return META_NS::IAny::Ptr {};
547 }
548 // hmm.. this is somewhat uncool
549 {
550 auto p1 = doc->GetProperty<IntfPtr>("EngineQueue");
551 doc->RemoveProperty(p1);
552 auto p2 = doc->GetProperty<IntfPtr>("AppQueue");
553 doc->RemoveProperty(p2);
554 auto p3 = doc->GetProperty<IntfPtr>("RenderContext");
555 doc->RemoveProperty(p3);
556 }
557
558 doc->GetArrayProperty<IntfWeakPtr>("Scenes")->Reset();
559 engineInstance_.applicationContext_.reset();
560 engineInstance_.renderContext_.reset();
561 engineInstance_.engine_.reset();
562
563 return META_NS::IAny::Ptr{};
564 });
565 engineThread->AddWaitableTask(engine_deinit)->Wait();
566 auto &tr = META_NS::GetTaskQueueRegistry();
567 tr.UnregisterTaskQueue(ENGINE_THREAD);
568 engineThread.reset();
569 tr.UnregisterTaskQueue(IO_QUEUE);
570 ioThread.reset();
571 tr.UnregisterTaskQueue(JS_RELEASE_THREAD);
572 releaseThread.reset();
573 }
574
ShutdownPluginRegistry()575 void SceneAdapter::ShutdownPluginRegistry()
576 {
577 if (engineInstance_.libHandle_ == nullptr) {
578 return;
579 }
580 dlclose(engineInstance_.libHandle_);
581 engineInstance_.libHandle_ = nullptr;
582
583 CORE_NS::GetPluginRegister = nullptr;
584 CORE_NS::CreatePluginRegistry = nullptr;
585 CORE_NS::IsDebugBuild = nullptr;
586 CORE_NS::GetVersion = nullptr;
587 }
588
PropSync()589 void SceneAdapter::PropSync()
590 {
591 auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_);
592 if (!scene) {
593 return;
594 }
595 auto internal = scene->GetInternalScene();
596 if (!internal) {
597 return;
598 }
599 internal->SyncProperties();
600 }
601
RenderFunction()602 void SceneAdapter::RenderFunction()
603 {
604 WIDGET_SCOPED_TRACE("SceneAdapter::RenderFunction");
605 auto rc = engineInstance_.renderContext_;
606 RETURN_IF_NULL(rc);
607 RETURN_IF_NULL(sceneWidgetObj_);
608 auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_);
609 RETURN_IF_NULL(scene);
610 BASE_NS::vector<SCENE_NS::ICamera::Ptr> disabledCameras;
611 auto cams = scene->GetCameras().GetResult();
612 for (auto c : cams) {
613 if (!bitmap_ && c->IsActive()) {
614 c->SetActive(false);
615 disabledCameras.push_back(c);
616 }
617
618 AttachSwapchain(interface_pointer_cast<META_NS::IObject>(c));
619 }
620
621 scene->GetInternalScene()->Update(false);
622 bool receiveBuffer = receiveBuffer_;
623 receiveBuffer_ = false;
624
625 auto& renderFrameUtil = rc->GetRenderUtil().GetRenderFrameUtil();
626 auto backendType = rc->GetDevice().GetBackendType();
627 if (receiveBuffer) {
628 if (sfBufferInfo_.acquireFence_) {
629 int32_t ret = sfBufferInfo_.acquireFence_->Wait(TIME_OUT_MILLISECONDS);
630 if (sfBufferInfo_.acquireFence_->Get() >= 0 && ret < 0) {
631 WIDGET_LOGE("wait fence error: %d", ret);
632 }
633 sfBufferInfo_.acquireFence_ = nullptr;
634 }
635 if (backendType == RENDER_NS::DeviceBackendType::VULKAN) {
636 RENDER_NS::IRenderFrameUtil::SignalData signalData;
637 signalData.signaled = false;
638 signalData.signalResourceType = RENDER_NS::IRenderFrameUtil::SignalResourceType::GPU_SEMAPHORE;
639 signalData.handle = bufferFenceHandle_;
640 renderFrameUtil.AddGpuSignal(signalData);
641 } else if (backendType == RENDER_NS::DeviceBackendType::OPENGLES) {
642 RENDER_NS::IRenderFrameUtil::SignalData signalData;
643 signalData.signaled = false;
644 signalData.signalResourceType = RENDER_NS::IRenderFrameUtil::SignalResourceType::GPU_FENCE;
645 signalData.handle = bufferFenceHandle_;
646 renderFrameUtil.AddGpuSignal(signalData);
647 }
648 }
649 rc->GetRenderer().RenderDeferredFrame();
650
651 for (auto& camera : disabledCameras) {
652 camera->SetActive(true);
653 }
654
655 if (receiveBuffer) {
656 auto addedSD = renderFrameUtil.GetFrameGpuSignalData();
657 for (const auto& ref : addedSD) {
658 if (ref.handle.GetHandle().id == bufferFenceHandle_.GetHandle().id) {
659 int32_t fenceFileDesc = CreateFenceFD(ref, rc->GetDevice());
660 sfBufferInfo_.fn_(fenceFileDesc);
661 break;
662 }
663 }
664 }
665 }
666
CreateFenceFD(const RENDER_NS::IRenderFrameUtil::SignalData & signalData,RENDER_NS::IDevice & device)667 int32_t SceneAdapter::CreateFenceFD(
668 const RENDER_NS::IRenderFrameUtil::SignalData &signalData, RENDER_NS::IDevice &device)
669 {
670 int32_t fenceFileDesc = -1;
671 auto backendType = device.GetBackendType();
672 if (backendType == RENDER_NS::DeviceBackendType::VULKAN) {
673 // NOTE: vulkan not implemented yet
674 WIDGET_LOGD("vulkan fence");
675 } else if (backendType == RENDER_NS::DeviceBackendType::OPENGLES) {
676 const auto disp = static_cast<const RENDER_NS::DevicePlatformDataGLES &>(device.GetPlatformData()).display;
677 EGLSyncKHR sync = reinterpret_cast<EGLSyncKHR>(static_cast<uintptr_t>(signalData.gpuSignalResourceHandle));
678 if (sync == EGL_NO_SYNC_KHR) {
679 WIDGET_LOGE("invalid EGLSync from signal data");
680 return fenceFileDesc;
681 }
682 fenceFileDesc = eglDupNativeFenceFDANDROID(disp, sync);
683 if (fenceFileDesc == -1) {
684 WIDGET_LOGE("eglDupNativeFence fail");
685 }
686 } else {
687 WIDGET_LOGE("no supported backend");
688 }
689 return fenceFileDesc;
690 }
691
RenderFrame(bool needsSyncPaint)692 void SceneAdapter::RenderFrame(bool needsSyncPaint)
693 {
694 if (!engineThread) {
695 WIDGET_LOGE("no engineThread for Render");
696 return;
697 }
698 if (renderTask) {
699 engineThread->CancelTask(renderTask);
700 renderTask = nullptr;
701 }
702
703 if (!singleFrameAsync_ || !singleFrameSync_ || !propSyncSync_) {
704 CreateRenderFunction();
705 }
706 if (!onWindowChanged_) {
707 WIDGET_LOGW("window has not changed");
708 return;
709 }
710 if (propSyncSync_) {
711 WIDGET_SCOPED_TRACE("SceneAdapter::propSyncSync_");
712 engineThread->AddWaitableTask(propSyncSync_)->Wait();
713 }
714
715 if (!needsSyncPaint && singleFrameAsync_) {
716 renderTask = engineThread->AddTask(singleFrameAsync_);
717 } else if (singleFrameSync_) {
718 engineThread->AddWaitableTask(singleFrameSync_)->Wait();
719 } else {
720 WIDGET_LOGE("No render function available.");
721 }
722 }
723
NeedsRepaint()724 bool SceneAdapter::NeedsRepaint()
725 {
726 return needsRepaint_;
727 }
728
SetNeedsRepaint(bool needsRepaint)729 void SceneAdapter::SetNeedsRepaint(bool needsRepaint)
730 {
731 needsRepaint_ = needsRepaint;
732 }
733
Deinit()734 void SceneAdapter::Deinit()
735 {
736 RETURN_IF_NULL(engineThread);
737 WIDGET_LOGI("SceneAdapter::Deinit");
738 auto func = META_NS::MakeCallback<META_NS::ITaskQueueWaitableTask>([this]() {
739 if (swapchainHandle_) {
740 auto& device = engineInstance_.renderContext_->GetDevice();
741 device.DestroySwapchain(swapchainHandle_);
742 }
743 swapchainHandle_ = {};
744 if (bitmap_) {
745 bitmap_.reset();
746 auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_);
747 if (!scene) {
748 return META_NS::IAny::Ptr {};
749 }
750 }
751 return META_NS::IAny::Ptr {};
752 });
753 engineThread->AddWaitableTask(func)->Wait();
754
755 sceneWidgetObj_.reset();
756 singleFrameAsync_.reset();
757 singleFrameSync_.reset();
758 propSyncSync_.reset();
759 needsRepaint_ = false;
760 onWindowChanged_ = false;
761 }
762
AttachSwapchain(META_NS::IObject::Ptr cameraObj)763 void SceneAdapter::AttachSwapchain(META_NS::IObject::Ptr cameraObj)
764 {
765 auto camera = interface_pointer_cast<SCENE_NS::ICamera>(cameraObj);
766 if (!camera) {
767 WIDGET_LOGE("cast cameraObj failed in AttachSwapchain.");
768 return;
769 }
770 if (!bitmap_ || !camera->IsActive()) {
771 camera->SetRenderTarget({});
772 return;
773 }
774 camera->SetRenderTarget(bitmap_);
775 }
776
777 namespace {
CreateOESTextureHandle(CORE_NS::IEcs::Ptr ecs,BASE_NS::shared_ptr<RENDER_NS::IRenderContext> renderContext,const SurfaceBufferInfo & surfaceBufferInfo)778 RENDER_NS::RenderHandleReference CreateOESTextureHandle(CORE_NS::IEcs::Ptr ecs,
779 BASE_NS::shared_ptr<RENDER_NS::IRenderContext> renderContext, const SurfaceBufferInfo &surfaceBufferInfo)
780 {
781 if (!ecs || !renderContext) {
782 WIDGET_LOGE("null ecs or renderContext");
783 return {};
784 }
785 auto surfaceBuffer = surfaceBufferInfo.sfBuffer_;
786 if (!surfaceBuffer) {
787 WIDGET_LOGE("surface buffer null");
788 return {};
789 }
790 auto nativeBuffer = surfaceBuffer->SurfaceBufferToNativeBuffer();
791 if (!nativeBuffer) {
792 WIDGET_LOGE("surface buffer to native buffer fail");
793 return {};
794 }
795
796 std::shared_ptr<RENDER_NS::BackendSpecificImageDesc> data = nullptr;
797 auto backendType = renderContext->GetDevice().GetBackendType();
798 if (backendType == RENDER_NS::DeviceBackendType::VULKAN) {
799 data = std::make_shared<RENDER_NS::ImageDescVk>();
800 auto vkData = std::static_pointer_cast<RENDER_NS::ImageDescVk>(data);
801 vkData->platformHwBuffer = reinterpret_cast<uintptr_t>(nativeBuffer);
802 } else if (backendType == RENDER_NS::DeviceBackendType::OPENGLES) {
803 data = std::make_shared<RENDER_NS::ImageDescGLES>();
804 auto glesData = std::static_pointer_cast<RENDER_NS::ImageDescGLES>(data);
805 glesData->type = GL_TEXTURE_EXTERNAL_OES;
806 glesData->platformHwBuffer = reinterpret_cast<uintptr_t>(nativeBuffer);
807 }
808
809 if (!data) {
810 WIDGET_LOGE("create ImageDesc based on backend fail!");
811 return {};
812 }
813
814 auto &gpuResourceManager = renderContext->GetDevice().GetGpuResourceManager();
815 auto handle = gpuResourceManager.CreateView("AR_CAMERA_PREVIEW", {}, *data);
816
817 return handle;
818 }
819 } // namespace
820
InitEnvironmentResource(const uint32_t bufferSize)821 void SceneAdapter::InitEnvironmentResource(const uint32_t bufferSize)
822 {
823 WIDGET_LOGI("Init resource for camera view");
824 auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_);
825 RETURN_IF_NULL(scene);
826 auto internalScene = scene->GetInternalScene();
827 RETURN_IF_NULL(internalScene);
828 auto ecs = internalScene->GetEcsContext().GetNativeEcs();
829 RETURN_IF_NULL(ecs);
830 auto renderContext = engineInstance_.renderContext_;
831 RETURN_IF_NULL(renderContext);
832
833 bufferFenceHandle_ = RENDER_NS::RenderHandleReference{{static_cast<uint64_t>(key_) + BUFFER_FENCE_HANDLE_BASE}, {}};
834
835 auto envManager = CORE_NS::GetManager<CORE3D_NS::IEnvironmentComponentManager>(*ecs);
836 RETURN_IF_NULL(envManager);
837 const auto &shaderMgr = renderContext->GetDevice().GetShaderManager();
838 auto &entityManager = (*ecs).GetEntityManager();
839 auto renderHandleManager = CORE_NS::GetManager<CORE3D_NS::IRenderHandleComponentManager>(*ecs);
840 RETURN_IF_NULL(renderHandleManager);
841 auto uriManager = CORE_NS::GetManager<CORE3D_NS::IUriComponentManager>(*ecs);
842 RETURN_IF_NULL(uriManager);
843
844 CORE_NS::EntityReference shaderEF = entityManager.CreateReferenceCounted();
845 renderHandleManager->Create(shaderEF);
846 BASE_NS::string_view shaderUri = "campreviewshaders://shader/camera_stream.shader";
847 renderContext->GetDevice().GetShaderManager().LoadShaderFile(shaderUri.data());
848 if (auto renderEntityHandle = renderHandleManager->Write(shaderEF)) {
849 renderEntityHandle->reference = shaderMgr.GetShaderHandle(shaderUri);
850 }
851 uriManager->Create(shaderEF);
852 if (auto uriHandle = uriManager->Write(shaderEF)) {
853 uriHandle->uri = shaderUri;
854 }
855
856 RENDER_NS::GpuBufferDesc envBufferDesc{
857 RENDER_NS::CORE_BUFFER_USAGE_UNIFORM_BUFFER_BIT | RENDER_NS::CORE_BUFFER_USAGE_TRANSFER_DST_BIT,
858 RENDER_NS::CORE_MEMORY_PROPERTY_HOST_VISIBLE_BIT | RENDER_NS::CORE_MEMORY_PROPERTY_HOST_COHERENT_BIT,
859 RENDER_NS::CORE_ENGINE_BUFFER_CREATION_DYNAMIC_RING_BUFFER,
860 bufferSize};
861 BASE_NS::array_view<const uint8_t> transData(
862 reinterpret_cast<const uint8_t *>(sfBufferInfo_.transformMatrix_.data()), bufferSize);
863 camTransBufferHandle_ =
864 renderContext->GetDevice().GetGpuResourceManager().Create(ENV_PROPERTIES_BUFFER_NAME, envBufferDesc, transData);
865 CORE_NS::EntityReference transBufferEF = entityManager.CreateReferenceCounted();
866 renderHandleManager->Create(transBufferEF);
867 if (auto renderEntityHandle = renderHandleManager->Write(transBufferEF)) {
868 renderEntityHandle->reference = camTransBufferHandle_;
869 }
870
871 CORE_NS::EntityReference transBufferEF2 = entityManager.CreateReferenceCounted();
872 renderHandleManager->Create(transBufferEF2);
873 if (auto renderEntityHandle = renderHandleManager->Write(transBufferEF2)) {
874 renderEntityHandle->reference = camTransBufferHandle_;
875 }
876
877 CORE_NS::EntityReference transBufferEF3 = entityManager.CreateReferenceCounted();
878 renderHandleManager->Create(transBufferEF3);
879 if (auto renderEntityHandle = renderHandleManager->Write(transBufferEF3)) {
880 renderEntityHandle->reference = camTransBufferHandle_;
881 }
882
883 camImageEF_ = entityManager.CreateReferenceCounted();
884 renderHandleManager->Create(camImageEF_);
885
886 if (auto rc = scene->RenderConfiguration()->GetValue()) {
887 if (auto env = rc->Environment()->GetValue()) {
888 CORE_NS::Entity environmentEntity = interface_pointer_cast<SCENE_NS::IEcsObjectAccess>(env)->
889 GetEcsObject()->GetEntity();
890 if (auto envDataHandle = envManager->Write(environmentEntity)) {
891 CORE3D_NS::EnvironmentComponent &envComponent = *envDataHandle;
892 envComponent.background = CORE3D_NS::EnvironmentComponent::Background::IMAGE;
893 envComponent.shader = shaderEF;
894 envComponent.customResources.push_back({camImageEF_});
895 envComponent.customResources.push_back({transBufferEF});
896 envComponent.customResources.push_back({transBufferEF2});
897 envComponent.customResources.push_back({transBufferEF3});
898 }
899 }
900 }
901 if (!renderDataStoreDefaultStaging_) {
902 renderDataStoreDefaultStaging_ = BASE_NS::refcnt_ptr<RENDER_NS::IRenderDataStoreDefaultStaging>(
903 static_cast<RENDER_NS::IRenderDataStoreDefaultStaging *>(
904 renderContext->GetRenderDataStoreManager().GetRenderDataStore(RENDER_DATA_STORE_DEFAULT_STAGING).get()));
905 RETURN_IF_NULL(renderDataStoreDefaultStaging_);
906 }
907 }
908
UpdateSurfaceBuffer()909 void SceneAdapter::UpdateSurfaceBuffer()
910 {
911 auto scene = interface_pointer_cast<SCENE_NS::IScene>(sceneWidgetObj_);
912 RETURN_IF_NULL(scene);
913 auto internalScene = scene->GetInternalScene();
914 RETURN_IF_NULL(internalScene);
915 auto ecs = internalScene->GetEcsContext().GetNativeEcs();
916 RETURN_IF_NULL(ecs);
917 auto renderContext = engineInstance_.renderContext_;
918 RETURN_IF_NULL(renderContext);
919
920 auto viewHandle = CreateOESTextureHandle(ecs, renderContext, sfBufferInfo_);
921 RETURN_IF_NULL(viewHandle);
922
923 const uint32_t bufferSize = sfBufferInfo_.transformMatrix_.size() * FLOAT_TO_BYTE;
924 if (bufferSize < TRANSFORM_MATRIX_SIZE * FLOAT_TO_BYTE) {
925 WIDGET_LOGE("transform matrix size:%d not 4x4", bufferSize / FLOAT_TO_BYTE);
926 const float *arr = BASE_NS::Math::IDENTITY_4X4.data;
927 sfBufferInfo_.transformMatrix_.assign(arr, arr + TRANSFORM_MATRIX_SIZE);
928 } else if (bufferSize > TRANSFORM_MATRIX_SIZE * FLOAT_TO_BYTE) {
929 WIDGET_LOGE("transform matrix size:%d not 4x4", bufferSize / FLOAT_TO_BYTE);
930 }
931
932 if (!initCamRNG_) {
933 InitEnvironmentResource(bufferSize);
934 initCamRNG_ = true;
935 }
936
937 if (auto renderHandleManager = CORE_NS::GetManager<CORE3D_NS::IRenderHandleComponentManager>(*ecs)) {
938 if (auto camImageRenderHandle = renderHandleManager->Write(camImageEF_)) {
939 camImageRenderHandle->reference = viewHandle;
940 }
941 }
942
943 if (camTransBufferHandle_ && renderDataStoreDefaultStaging_) {
944 BASE_NS::array_view<const uint8_t> transData(
945 reinterpret_cast<const uint8_t *>(sfBufferInfo_.transformMatrix_.data()), bufferSize);
946 const RENDER_NS::BufferCopy bufferCopy{0, 0, bufferSize};
947 renderDataStoreDefaultStaging_->CopyDataToBufferOnCpu(transData, camTransBufferHandle_, bufferCopy);
948 }
949 }
950
AcquireImage(const SurfaceBufferInfo & bufferInfo)951 void SceneAdapter::AcquireImage(const SurfaceBufferInfo &bufferInfo)
952 {
953 RETURN_IF_NULL(engineThread);
954 engineThread->AddTask(META_NS::MakeCallback<META_NS::ITaskQueueTask>([this, bufferInfo]() {
955 sfBufferInfo_ = bufferInfo;
956 receiveBuffer_ = true;
957 UpdateSurfaceBuffer();
958 return false;
959 }));
960 }
961 } // namespace OHOS::Render3D
962