1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "js_third_provider_interaction_operation.h"
17
18 #include "accessibility_system_ability_client.h"
19
20 #include "core/pipeline_ng/pipeline_context.h"
21 #include "js_third_provider_interaction_operation_utils.h"
22
23 namespace OHOS::Ace::Framework {
24 namespace {
LogAccessibilityElementInfo(const std::string & tag,const Accessibility::AccessibilityElementInfo & info)25 void LogAccessibilityElementInfo(
26 const std::string& tag, const Accessibility::AccessibilityElementInfo& info)
27 {
28 std::string printStr;
29 printStr.append(tag).append(", info: [");
30 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
31 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
32 printStr.append(", childCount: ").append(std::to_string(info.GetChildCount()));
33 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
34 printStr.append(", parentNodeId: ").append(std::to_string(info.GetParentNodeId()));
35 printStr.append(", checkable: ").append(info.IsCheckable() ? "true" : "false");
36 printStr.append(", checked: ").append(info.IsChecked() ? "true" : "false");
37 printStr.append(", focusable: ").append(info.IsFocusable() ? "true" : "false");
38 printStr.append(", focused: ").append(info.IsFocused() ? "true" : "false");
39 printStr.append(", visible: ").append(info.IsVisible() ? "true" : "false");
40 printStr.append(", hasAccessibilityFocus: ")
41 .append(info.HasAccessibilityFocus() ? "true" : "false");
42 printStr.append(", selected: ").append(info.IsSelected() ? "true" : "false");
43 printStr.append(", clickable: ").append(info.IsClickable() ? "true" : "false");
44 printStr.append(", longClickable: ").append(info.IsLongClickable() ? "true" : "false");
45 printStr.append(", enabled: ").append(info.IsEnabled() ? "true" : "false");
46 printStr.append(", componentType: ").append(info.GetComponentType());
47 printStr.append(", content: ").append(info.GetContent());
48 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
49 printStr.append(", triggerAction: ")
50 .append(std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
51 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
52 printStr.append(", childTreeId: ").append(std::to_string(info.GetChildTreeId()));
53 printStr.append(", belongTreeId: ").append(std::to_string(info.GetBelongTreeId()));
54 printStr.append(", parentWindowId: ").append(std::to_string(info.GetParentWindowId()));
55 printStr.append(", accessibilityGroup: ").append(info.GetAccessibilityGroup() ? "true" : "false");
56 std::string actionStr;
57 actionStr.append("{");
58 for (const auto& action : info.GetActionList()) {
59 actionStr.append(std::to_string(
60 static_cast<int32_t>(action.GetActionType()))).append(" ");
61 }
62 actionStr.append("}");
63 printStr.append(", action: ").append(actionStr);
64 printStr.append("]");
65 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
66 }
67
LogAccessibilityEventInfo(const std::string & tag,const Accessibility::AccessibilityEventInfo & info)68 void LogAccessibilityEventInfo(
69 const std::string& tag, const Accessibility::AccessibilityEventInfo& info)
70 {
71 std::string printStr;
72 printStr.append(tag).append(", info: [");
73 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
74 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
75 printStr.append(", viewId: ").append(std::to_string(info.GetViewId()));
76 printStr.append(", componentType: ").append(info.GetComponentType());
77 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
78 printStr.append(", eventType: ").append(
79 std::to_string(static_cast<int32_t>(info.GetEventType())));
80 printStr.append(", triggerAction: ").append(
81 std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
82 printStr.append(", requestFocusElementId: ").append(
83 std::to_string(info.GetRequestFocusElementId()));
84 printStr.append("]");
85 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
86 }
FillNodeConfig(const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)87 void FillNodeConfig(
88 const NodeConfig& config, Accessibility::AccessibilityElementInfo& info)
89 {
90 OHOS::Accessibility::Rect oldRect = info.GetRectInScreen();
91 OHOS::Accessibility::Rect newRect = OHOS::Accessibility::Rect(
92 oldRect.GetLeftTopXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
93 oldRect.GetLeftTopYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()),
94 oldRect.GetRightBottomXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
95 oldRect.GetRightBottomYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()));
96 info.SetRectInScreen(newRect);
97 info.SetPageId(config.pageId);
98 info.SetWindowId(config.windowId);
99 info.SetBelongTreeId(config.belongTreeId);
100 info.SetChildTreeIdAndWinId(config.childTreeId, config.childWindowId);
101 info.SetParentWindowId(config.parentWindowId);
102 info.SetBundleName(config.bundleName);
103 info.SetInspectorKey(config.inspectorKey);
104 int64_t splitElementId = info.GetAccessibilityId();
105 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(config.belongTreeId, splitElementId);
106 info.SetAccessibilityId(splitElementId);
107 }
108
CopyNativeInfoToAccessibilityElementInfo(const ArkUI_AccessibilityElementInfo & elementInfo,const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)109 void CopyNativeInfoToAccessibilityElementInfo(
110 const ArkUI_AccessibilityElementInfo& elementInfo,
111 const NodeConfig& config,
112 Accessibility::AccessibilityElementInfo& info)
113 {
114 TransformAccessbilityElementInfo(elementInfo, info);
115 FillNodeConfig(config, info);
116 }
117
CopyNativeInfosToAccessibilityElementInfos(const std::vector<ArkUI_AccessibilityElementInfo> & nativeInfos,const NodeConfig & config,std::list<Accessibility::AccessibilityElementInfo> & infos)118 void CopyNativeInfosToAccessibilityElementInfos(
119 const std::vector<ArkUI_AccessibilityElementInfo>& nativeInfos,
120 const NodeConfig& config,
121 std::list<Accessibility::AccessibilityElementInfo>& infos)
122 {
123 for (const auto& nativeInfo : nativeInfos) {
124 Accessibility::AccessibilityElementInfo info;
125 TransformAccessbilityElementInfo(nativeInfo, info);
126 FillNodeConfig(config, info);
127 infos.push_back(info);
128 }
129 }
130
CheckEventIgnoreHostOffset(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent)131 bool CheckEventIgnoreHostOffset(
132 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent)
133 {
134 auto eventType = nativeAccessibilityEvent.GetEventType();
135 bool ignoreHostOffset = false;
136 switch (eventType) {
137 case ArkUI_AccessibilityEventType::
138 ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
139 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by focus update");
140 ignoreHostOffset = true;
141 break;
142 default:
143 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by other eventType");
144 }
145 return ignoreHostOffset;
146 }
147 } // namespace
148
JsThirdProviderInteractionOperation(const WeakPtr<AccessibilityProvider> & accessibilityProvider,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager,WeakPtr<NG::FrameNode> host)149 JsThirdProviderInteractionOperation::JsThirdProviderInteractionOperation(
150 const WeakPtr<AccessibilityProvider>& accessibilityProvider,
151 const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager,
152 WeakPtr<NG::FrameNode> host)
153 : accessibilityProvider_(accessibilityProvider),
154 jsAccessibilityManager_(jsAccessibilityManager),
155 host_(host)
156 {
157 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Create");
158 }
159
~JsThirdProviderInteractionOperation()160 JsThirdProviderInteractionOperation::~JsThirdProviderInteractionOperation()
161 {
162 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Destory");
163 }
164
Initialize()165 void JsThirdProviderInteractionOperation::Initialize()
166 {
167 auto provider = accessibilityProvider_.Upgrade();
168 CHECK_NULL_VOID(provider);
169 }
170
SearchElementInfoByAccessibilityId(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t mode)171 void JsThirdProviderInteractionOperation::SearchElementInfoByAccessibilityId(
172 const int64_t elementId, const int32_t requestId,
173 Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode)
174 {
175 uint32_t realMode = mode;
176 if (realMode & static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED)) {
177 realMode &= ~static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED);
178 realMode |= static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN);
179 }
180 // 1. Get real elementId
181 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
182 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
183 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
184 elementId, splitElementId, splitTreeId);
185
186 // 2. FindAccessibilityNodeInfosById by provider
187 std::list<Accessibility::AccessibilityElementInfo> infos;
188 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
189 splitElementId, realMode, requestId, infos);
190 if (!ret) {
191 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
192 "SearchElementInfoByAccessibilityId failed.");
193 infos.clear();
194 }
195
196 // 3. Return result
197 SetSearchElementInfoByAccessibilityIdResult(callback, std::move(infos), requestId);
198 }
199
FindAccessibilityNodeInfosByIdFromProvider(const int64_t splitElementId,const int32_t mode,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos,bool ignoreHostOffset)200 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByIdFromProvider(
201 const int64_t splitElementId, const int32_t mode, const int32_t requestId,
202 std::list<Accessibility::AccessibilityElementInfo>& infos, bool ignoreHostOffset)
203 {
204 auto provider = accessibilityProvider_.Upgrade();
205 CHECK_NULL_RETURN(provider, false);
206 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
207 int32_t code = provider->FindAccessibilityNodeInfosById(
208 splitElementId, mode, requestId, nativeInfos);
209 if (code != 0) {
210 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
211 "FindAccessibilityNodeInfosByIdFromProvider failed: %{public}d", code);
212 return false;
213 }
214
215 NodeConfig config;
216 config.ignoreHostOffset = ignoreHostOffset;
217 GetNodeConfig(config);
218 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
219 return !infos.empty();
220 }
221
SetSearchElementInfoByAccessibilityIdResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)222 void JsThirdProviderInteractionOperation::SetSearchElementInfoByAccessibilityIdResult(
223 AccessibilityElementOperatorCallback& callback,
224 std::list<AccessibilityElementInfo>&& infos,
225 const int32_t requestId)
226 {
227 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
228 CHECK_NULL_VOID(jsAccessibilityManager);
229 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
230 CHECK_NULL_VOID(context);
231 CHECK_NULL_VOID(context->GetTaskExecutor());
232 context->GetTaskExecutor()->PostTask(
233 [jsMgr = jsAccessibilityManager_,
234 infos = std::move(infos), &callback, requestId] () mutable {
235 auto jsAccessibilityManager = jsMgr.Upgrade();
236 CHECK_NULL_VOID(jsAccessibilityManager);
237 if (!jsAccessibilityManager->IsRegister()) {
238 return;
239 }
240 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
241 callback.SetSearchElementInfoByAccessibilityIdResult(infos, requestId);
242 }, TaskExecutor::TaskType::BACKGROUND, "SearchElementInfoByAccessibilityId");
243 }
244
SearchElementInfosByText(const int64_t elementId,const std::string & text,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)245 void JsThirdProviderInteractionOperation::SearchElementInfosByText(
246 const int64_t elementId, const std::string& text, const int32_t requestId,
247 Accessibility::AccessibilityElementOperatorCallback& callback)
248 {
249 // 1. Get real elementId
250 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
251 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
252 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
253 elementId, splitElementId, splitTreeId);
254
255 // 2. FindAccessibilityNodeInfosByText from provider
256 std::list<Accessibility::AccessibilityElementInfo> infos;
257 bool ret = FindAccessibilityNodeInfosByTextFromProvider(
258 splitElementId, text, requestId, infos);
259 if (!ret) {
260 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
261 "SearchElementInfosByText failed.");
262 infos.clear();
263 }
264
265 // 3. Return result
266 SetSearchElementInfoByTextResult(callback, std::move(infos), requestId);
267 }
268
SearchDefaultFocusByWindowId(const int32_t windowId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t pageId)269 void JsThirdProviderInteractionOperation::SearchDefaultFocusByWindowId(
270 const int32_t windowId, const int32_t requestId,
271 Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t pageId)
272 {
273 }
274
FindAccessibilityNodeInfosByTextFromProvider(const int64_t splitElementId,const std::string & text,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos)275 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByTextFromProvider(
276 const int64_t splitElementId, const std::string& text, const int32_t requestId,
277 std::list<Accessibility::AccessibilityElementInfo>& infos)
278 {
279 auto provider = accessibilityProvider_.Upgrade();
280 CHECK_NULL_RETURN(provider, false);
281 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
282 int32_t code = provider->FindAccessibilityNodeInfosByText(
283 splitElementId, text, requestId, nativeInfos);
284 if (code != 0) {
285 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
286 "FindAccessibilityNodeInfosByTextFromProvider failed: %{public}d", code);
287 return false;
288 }
289
290 NodeConfig config;
291 GetNodeConfig(config);
292 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
293 return true;
294 }
295
SetSearchElementInfoByTextResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)296 void JsThirdProviderInteractionOperation::SetSearchElementInfoByTextResult(
297 AccessibilityElementOperatorCallback& callback,
298 std::list<AccessibilityElementInfo>&& infos,
299 const int32_t requestId)
300 {
301 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
302 CHECK_NULL_VOID(jsAccessibilityManager);
303 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
304 CHECK_NULL_VOID(context);
305 CHECK_NULL_VOID(context->GetTaskExecutor());
306 context->GetTaskExecutor()->PostTask(
307 [jsMgr = jsAccessibilityManager_,
308 infos = std::move(infos), &callback, requestId] () mutable {
309 auto jsAccessibilityManager = jsMgr.Upgrade();
310 CHECK_NULL_VOID(jsAccessibilityManager);
311 if (!jsAccessibilityManager->IsRegister()) {
312 return;
313 }
314 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
315 callback.SetSearchElementInfoByTextResult(infos, requestId);
316 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
317 }
318
FindFocusedElementInfo(const int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)319 void JsThirdProviderInteractionOperation::FindFocusedElementInfo(
320 const int64_t elementId, const int32_t focusType, const int32_t requestId,
321 Accessibility::AccessibilityElementOperatorCallback& callback)
322 {
323 // 1. Get real elementId
324 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
325 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
326 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
327 elementId, splitElementId, splitTreeId);
328
329 // 2. FindFocusedAccessibilityNode from provider
330 Accessibility::AccessibilityElementInfo info;
331 bool ret = FindFocusedElementInfoFromProvider(
332 splitElementId, focusType, requestId, info);
333 if (!ret) {
334 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
335 "SearchElementInfosByText failed.");
336 info.SetValidElement(false);
337 }
338
339 // 3. Return result
340 SetFindFocusedElementInfoResult(callback, info, requestId);
341 }
342
FindFocusedElementInfoFromProvider(int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)343 bool JsThirdProviderInteractionOperation::FindFocusedElementInfoFromProvider(
344 int64_t elementId, const int32_t focusType, const int32_t requestId,
345 Accessibility::AccessibilityElementInfo& info)
346 {
347 auto provider = accessibilityProvider_.Upgrade();
348 CHECK_NULL_RETURN(provider, false);
349 ArkUI_AccessibilityElementInfo nativeInfo;
350 int32_t code = provider->FindFocusedAccessibilityNode(
351 elementId, focusType, requestId, nativeInfo);
352 if (code != 0) {
353 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
354 "FindFocusedElementInfoFromProvider failed: %{public}d", code);
355 return false;
356 }
357
358 NodeConfig config;
359 GetNodeConfig(config);
360 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
361 return true;
362 }
363
SetFindFocusedElementInfoResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)364 void JsThirdProviderInteractionOperation::SetFindFocusedElementInfoResult(
365 AccessibilityElementOperatorCallback& callback,
366 AccessibilityElementInfo& info,
367 const int32_t requestId)
368 {
369 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
370 CHECK_NULL_VOID(jsAccessibilityManager);
371 jsAccessibilityManager->UpdateElementInfoTreeId(info);
372 callback.SetFindFocusedElementInfoResult(info, requestId);
373 }
374
FocusMoveSearch(const int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)375 void JsThirdProviderInteractionOperation::FocusMoveSearch(
376 const int64_t elementId, const int32_t direction, const int32_t requestId,
377 Accessibility::AccessibilityElementOperatorCallback& callback)
378 {
379 // 1. Get real elementId
380 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
381 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
382 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
383 elementId, splitElementId, splitTreeId);
384
385 // 2. FindNextFocusAccessibilityNode from provider
386 Accessibility::AccessibilityElementInfo info;
387 bool ret = FocusMoveSearchProvider(
388 splitElementId, direction, requestId, info);
389 // 3. Return result
390 if (!ret) {
391 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
392 CHECK_NULL_VOID(jsAccessibilityManager);
393 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
394 CHECK_NULL_VOID(context);
395 CHECK_NULL_VOID(context->GetTaskExecutor());
396 context->GetTaskExecutor()->PostTask(
397 [jsMgr = jsAccessibilityManager_, weakHost = host_, &callback, requestId] () {
398 auto jsAccessibilityManager = jsMgr.Upgrade();
399 if ((!jsAccessibilityManager) || (!jsAccessibilityManager->IsRegister())) {
400 AccessibilityElementInfo nodeInfo;
401 nodeInfo.SetValidElement(false);
402 callback.SetFocusMoveSearchResult(nodeInfo, requestId);
403 return;
404 }
405 jsAccessibilityManager->SetFocusMoveResultWithNode(weakHost, callback, requestId);
406 }, TaskExecutor::TaskType::UI, "AccessibilityThirdPartyFocusMoveSearchFail");
407 } else {
408 SetFocusMoveSearchResult(callback, info, requestId);
409 }
410 }
411
FocusMoveSearchProvider(int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)412 bool JsThirdProviderInteractionOperation::FocusMoveSearchProvider(
413 int64_t elementId, const int32_t direction, const int32_t requestId,
414 Accessibility::AccessibilityElementInfo& info)
415 {
416 auto provider = accessibilityProvider_.Upgrade();
417 CHECK_NULL_RETURN(provider, false);
418 ArkUI_AccessibilityElementInfo nativeInfo;
419 int32_t code = provider->FindNextFocusAccessibilityNode(
420 elementId, direction, requestId, nativeInfo);
421 if (code != 0) {
422 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
423 "FocusMoveSearchProvider failed: %{public}d", code);
424 return false;
425 }
426
427 NodeConfig config;
428 GetNodeConfig(config);
429 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
430 return true;
431 }
432
SetFocusMoveSearchResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)433 void JsThirdProviderInteractionOperation::SetFocusMoveSearchResult(
434 AccessibilityElementOperatorCallback& callback,
435 AccessibilityElementInfo& info,
436 const int32_t requestId)
437 {
438 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
439 CHECK_NULL_VOID(jsAccessibilityManager);
440 jsAccessibilityManager->UpdateElementInfoTreeId(info);
441 callback.SetFocusMoveSearchResult(info, requestId);
442 }
443
HandleActionWhenFindNodeFail(const int32_t action)444 void JsThirdProviderInteractionOperation::HandleActionWhenFindNodeFail(const int32_t action)
445 {
446 if (action == static_cast<int32_t>(
447 Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
448 ClearDrawBound();
449 }
450 }
451
ExecuteAction(const int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)452 void JsThirdProviderInteractionOperation::ExecuteAction(
453 const int64_t elementId, const int32_t action,
454 const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
455 Accessibility::AccessibilityElementOperatorCallback& callback)
456 {
457 // 1. Get real elementId
458 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
459 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
460 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
461 elementId, splitElementId, splitTreeId);
462
463 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAction elementId: %{public}" PRId64 ","
464 " action: %{public}d, requestId: %{public}d, splitElementId: %{public}" PRId64 ","
465 " splitTreeId: %{public}d",
466 elementId, action, requestId, splitElementId, splitTreeId);
467 // 2. FindNextFocusAccessibilityNode from provider
468 std::list<Accessibility::AccessibilityElementInfo> infos;
469 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
470 splitElementId, 0, requestId, infos, true); // for drawbound, no need fix host offset
471 if (!ret) {
472 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "Find info failed when ExecuteAction.");
473 SetExecuteActionResult(callback, false, requestId);
474 HandleActionWhenFindNodeFail(action);
475 return;
476 }
477
478 // 3. DrawBound
479 ExecuteActionForThird(splitElementId, infos.front(), action);
480
481 //4. ExecuteAccessibilityAction To provider
482 ret = ExecuteActionFromProvider(splitElementId, action, actionArguments, requestId);
483 if (!ret) {
484 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAccessibilityAction failed.");
485 }
486
487 // 5. Return result
488 SetExecuteActionResult(callback, ret, requestId);
489 }
490
ExecuteActionFromProvider(int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId)491 bool JsThirdProviderInteractionOperation::ExecuteActionFromProvider(
492 int64_t elementId, const int32_t action,
493 const std::map<std::string, std::string>& actionArguments, const int32_t requestId)
494 {
495 auto provider = accessibilityProvider_.Upgrade();
496 CHECK_NULL_RETURN(provider, false);
497 int32_t code = provider->ExecuteAccessibilityAction(
498 elementId, action, requestId, actionArguments);
499 if (code != 0) {
500 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
501 "ExecuteActionFromProvider failed: %{public}d", code);
502 return false;
503 }
504
505 return true;
506 }
507
ExecuteActionForThird(int64_t elementId,const AccessibilityElementInfo & nodeInfo,const int32_t action)508 bool JsThirdProviderInteractionOperation::ExecuteActionForThird(
509 int64_t elementId, const AccessibilityElementInfo& nodeInfo, const int32_t action)
510 {
511 auto hostNode = GetHost();
512 CHECK_NULL_RETURN(hostNode, false);
513 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
514 CHECK_NULL_RETURN(jsAccessibilityManager, false);
515 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
516 CHECK_NULL_RETURN(context, false);
517 auto ngPipeline = AceType::DynamicCast<NG::PipelineContext>(context);
518 CHECK_NULL_RETURN(ngPipeline, false);
519 if (action == static_cast<int32_t>(
520 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS)) {
521 jsAccessibilityManager->ActThirdAccessibilityFocus(
522 elementId, nodeInfo, hostNode, ngPipeline, false);
523 } else if (action == static_cast<int32_t>(
524 Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
525 jsAccessibilityManager->ActThirdAccessibilityFocus(
526 elementId, nodeInfo, hostNode, ngPipeline, true);
527 }
528 return true;
529 }
530
SetExecuteActionResult(AccessibilityElementOperatorCallback & callback,const bool succeeded,const int32_t requestId)531 void JsThirdProviderInteractionOperation::SetExecuteActionResult(
532 AccessibilityElementOperatorCallback& callback,
533 const bool succeeded, const int32_t requestId)
534 {
535 callback.SetExecuteActionResult(succeeded, requestId);
536 }
537
ClearFocus()538 void JsThirdProviderInteractionOperation::ClearFocus()
539 {
540 // 1. Clear focus from provider
541 ClearFocusFromProvider();
542 // 2. Clear DrawBound
543 ClearDrawBound();
544 }
545
ClearFocusFromProvider()546 bool JsThirdProviderInteractionOperation::ClearFocusFromProvider()
547 {
548 auto provider = accessibilityProvider_.Upgrade();
549 CHECK_NULL_RETURN(provider, false);
550 int32_t code = provider->ClearFocusedAccessibilityNode();
551 if (code != 0) {
552 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
553 "ExecuteActionFromProvider failed: %{public}d", code);
554 return false;
555 }
556 return true;
557 }
558
ClearDrawBound()559 bool JsThirdProviderInteractionOperation::ClearDrawBound()
560 {
561 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
562 CHECK_NULL_RETURN(jsAccessibilityManager, false);
563 auto hostNode = GetHost();
564 CHECK_NULL_RETURN(hostNode, false);
565 return jsAccessibilityManager->ClearThirdAccessibilityFocus(hostNode);
566 }
567
OutsideTouch()568 void JsThirdProviderInteractionOperation::OutsideTouch()
569 {}
570
GetCursorPosition(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)571 void JsThirdProviderInteractionOperation::GetCursorPosition(
572 const int64_t elementId, const int32_t requestId,
573 Accessibility::AccessibilityElementOperatorCallback &callback)
574 {
575 auto provider = accessibilityProvider_.Upgrade();
576 CHECK_NULL_VOID(provider);
577 int32_t cursorPosition = -1;
578 int32_t code = provider->GetAccessibilityNodeCursorPosition(
579 elementId, requestId, cursorPosition);
580 if (code != 0) {
581 callback.SetCursorPositionResult(-1, requestId);
582 return;
583 }
584
585 callback.SetCursorPositionResult(cursorPosition, requestId);
586 }
587
GetHostRectTranslateInfo(NodeConfig & config)588 void JsThirdProviderInteractionOperation::GetHostRectTranslateInfo(NodeConfig& config)
589 {
590 auto host = host_.Upgrade();
591 if ((!host) || config.ignoreHostOffset) {
592 config.offset = NG::OffsetF(0, 0);
593 config.scaleX = 1.0f;
594 config.scaleY = 1.0f;
595 return;
596 }
597 auto rect = host->GetTransformRectRelativeToWindow();
598 NG::VectorF finalScale = host->GetTransformScaleRelativeToWindow();
599 auto pipeline = host->GetContextRefPtr();
600 if (pipeline) {
601 auto accessibilityManager = pipeline->GetAccessibilityManager();
602 if (accessibilityManager) {
603 auto windowInfo = accessibilityManager->GenerateWindowInfo(host, pipeline);
604 auto top = rect.Top() * windowInfo.scaleY + static_cast<int32_t>(windowInfo.top);
605 auto left = rect.Left() * windowInfo.scaleX + static_cast<int32_t>(windowInfo.left);
606 finalScale.x *= windowInfo.scaleX;
607 finalScale.y *= windowInfo.scaleY;
608 config.offset = NG::OffsetT(left, top);
609 } else {
610 auto windowRect = pipeline->GetDisplayWindowRectInfo();
611 auto top = rect.Top() + static_cast<int32_t>(windowRect.Top());
612 auto left = rect.Left() + static_cast<int32_t>(windowRect.Left());
613 config.offset = NG::OffsetT(left, top);
614 }
615 }
616
617 config.scaleX = finalScale.x;
618 config.scaleY = finalScale.y;
619 }
620
GetNodeConfig(NodeConfig & config)621 void JsThirdProviderInteractionOperation::GetNodeConfig(NodeConfig& config)
622 {
623 auto host = host_.Upgrade();
624 CHECK_NULL_VOID(host);
625 auto jsAccessibilityManager = GetHandler().Upgrade();
626 CHECK_NULL_VOID(jsAccessibilityManager);
627 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
628 CHECK_NULL_VOID(context);
629 config.pageId = host->GetPageId();
630 config.windowId = static_cast<int32_t>(context->GetRealHostWindowId());
631 config.belongTreeId = belongTreeId_;
632 config.parentWindowId = static_cast<int32_t>(context->GetRealHostWindowId());
633 config.bundleName = AceApplicationInfo::GetInstance().GetPackageName();
634
635 GetHostRectTranslateInfo(config);
636 }
637
SetChildTreeIdAndWinId(const int64_t nodeId,const int32_t treeId,const int32_t childWindowId)638 void JsThirdProviderInteractionOperation::SetChildTreeIdAndWinId(
639 const int64_t nodeId, const int32_t treeId, const int32_t childWindowId)
640 {
641 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetChildTreeIdAndWinId, node: %{public}" PRId64 ","
642 "treeId: %{public}d, childWindowId: %{public}d", nodeId, treeId, childWindowId);
643 }
644
SetBelongTreeId(const int32_t treeId)645 void JsThirdProviderInteractionOperation::SetBelongTreeId(const int32_t treeId)
646 {
647 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetBelongTreeId treeId: %{public}d", treeId);
648 belongTreeId_ = treeId;
649 }
650
SendAccessibilityAsyncEventForThird(int64_t thirdElementId,Accessibility::EventType eventType)651 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEventForThird(
652 int64_t thirdElementId,
653 Accessibility::EventType eventType)
654 {
655 // 1. generate event
656 Accessibility::AccessibilityEventInfo event;
657 event.SetTimeStamp(GetMicroTickCount());
658 event.SetWindowChangeTypes(
659 Accessibility::WindowUpdateType::WINDOW_UPDATE_INVALID);
660 event.SetWindowContentChangeTypes(
661 Accessibility::WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_INVALID);
662 event.SetSource(thirdElementId);
663 event.SetEventType(eventType);
664 event.SetBundleName(AceApplicationInfo::GetInstance().GetPackageName());
665
666 // 2. get element from third
667 // cut tree id
668 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
669 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
670 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
671 thirdElementId, splitElementId, splitTreeId);
672 std::list<Accessibility::AccessibilityElementInfo> infos;
673 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
674 splitElementId, 0, 0, infos);
675 if ((!ret) || (infos.size() == 0)) {
676 return -1;
677 }
678
679 auto jsAccessibilityManager = GetHandler().Upgrade();
680 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
681 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
682 event.SetElementInfo(infos.front());
683
684 // 3. change event info by host info
685 GetAccessibilityEventInfoFromNativeEvent(event, false);
686
687 // 4. SendEvent
688 auto host = host_.Upgrade();
689 CHECK_NULL_RETURN(host, -1);
690 SendAccessibilitySyncEventToService(event, nullptr);
691 return 0;
692 }
693
HandleNativeFocusUpdate(int64_t elementId,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)694 bool JsThirdProviderInteractionOperation::HandleNativeFocusUpdate(
695 int64_t elementId,
696 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
697 {
698 ExecuteActionForThird(
699 elementId,
700 accessibilityEventInfo.GetElementInfo(),
701 static_cast<int32_t>(
702 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS));
703 return false;
704 }
705
HandleEventByFramework(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)706 bool JsThirdProviderInteractionOperation::HandleEventByFramework(
707 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
708 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
709 {
710 auto eventType = nativeAccessibilityEvent.GetEventType();
711 bool needSendEvent = true;
712 switch (eventType) {
713 case ArkUI_AccessibilityEventType::
714 ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
715 if (nativeAccessibilityEvent.GetElementInfo()) {
716 needSendEvent = HandleNativeFocusUpdate(
717 nativeAccessibilityEvent.GetElementInfo()->GetElementId(),
718 accessibilityEventInfo);
719 }
720 break;
721 default:
722 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Unsupported eventType");
723 }
724 return needSendEvent;
725 }
726
SendAccessibilityAsyncEvent(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,void (* callback)(int32_t errorCode))727 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEvent(
728 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
729 void (*callback)(int32_t errorCode))
730 {
731 bool ignoreHostOffset = CheckEventIgnoreHostOffset(nativeAccessibilityEvent);
732 // 1. Get OHOS::Accessibility::AccessibilityEventInfo
733 OHOS::Accessibility::AccessibilityEventInfo accessibilityEventInfo;
734 GetAccessibilityEventInfoFromNativeEvent(
735 nativeAccessibilityEvent, accessibilityEventInfo, ignoreHostOffset);
736
737 // 2. handleEvent by frame work
738 bool needSendEvent = HandleEventByFramework(
739 nativeAccessibilityEvent,
740 accessibilityEventInfo);
741 // 3. SendEvent
742 if (needSendEvent) {
743 auto jsAccessibilityManager = GetHandler().Upgrade();
744 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
745 auto host = host_.Upgrade();
746 CHECK_NULL_RETURN(host, -1);
747 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Inner SendAccessibilityAsyncEvent");
748 SendAccessibilitySyncEventToService(accessibilityEventInfo, callback);
749 }
750 callback(0);
751 return 0;
752 }
753
GetAccessibilityEventInfoFromNativeEvent(OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)754 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
755 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
756 bool ignoreHostOffset)
757 {
758 // 1. Fill node config
759 NodeConfig config;
760 config.ignoreHostOffset = ignoreHostOffset;
761 GetNodeConfig(config);
762
763 // 1.1. Fill elementInfo config
764 auto elementInfo = accessibilityEventInfo.GetElementInfo();
765 FillNodeConfig(config, elementInfo);
766 int64_t elementId = elementInfo.GetAccessibilityId();
767 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(belongTreeId_, elementId);
768 elementInfo.SetAccessibilityId(elementId);
769
770 // 1.2. Fill eventInfo config
771 accessibilityEventInfo.SetPageId(config.pageId);
772 accessibilityEventInfo.SetWindowId(config.windowId);
773 accessibilityEventInfo.SetSource(elementId);
774 accessibilityEventInfo.SetComponentType(elementInfo.GetComponentType());
775 accessibilityEventInfo.AddContent(elementInfo.GetContent());
776 accessibilityEventInfo.SetElementInfo(elementInfo);
777 LogAccessibilityElementInfo("sendEvent", elementInfo);
778 LogAccessibilityEventInfo("sendEvent", accessibilityEventInfo);
779 }
780
GetAccessibilityEventInfoFromNativeEvent(const ArkUI_AccessibilityEventInfo & nativeEventInfo,OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)781 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
782 const ArkUI_AccessibilityEventInfo& nativeEventInfo,
783 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
784 bool ignoreHostOffset)
785 {
786 // 1. Transform native event to OHOS::Accessibility::AccessibilityEventInfo
787 TransformAccessbilityEventInfo(
788 nativeEventInfo, accessibilityEventInfo);
789
790 // 2. Transform Accessibility::AccessibilityEventInfo with host info
791 GetAccessibilityEventInfoFromNativeEvent(accessibilityEventInfo, ignoreHostOffset);
792 }
793
SendAccessibilitySyncEventToService(const OHOS::Accessibility::AccessibilityEventInfo & eventInfo,void (* callback)(int32_t errorCode))794 bool JsThirdProviderInteractionOperation::SendAccessibilitySyncEventToService(
795 const OHOS::Accessibility::AccessibilityEventInfo& eventInfo,
796 [[maybe_unused]] void (*callback)(int32_t errorCode))
797 {
798 auto jsAccessibilityManager = GetHandler().Upgrade();
799 CHECK_NULL_RETURN(jsAccessibilityManager, false);
800 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
801 CHECK_NULL_RETURN(context, false);
802 CHECK_NULL_RETURN(context->GetTaskExecutor(), false);
803 context->GetTaskExecutor()->PostTask(
804 [jsMgr = jsAccessibilityManager_, eventInfo] () mutable {
805 auto jsAccessibilityManager = jsMgr.Upgrade();
806 if (jsAccessibilityManager == nullptr) {
807 return;
808 }
809
810 if (!jsAccessibilityManager->IsRegister()) {
811 return;
812 }
813
814 auto client = AccessibilitySystemAbilityClient::GetInstance();
815 CHECK_NULL_VOID(client);
816 bool isEnabled = false;
817 client->IsEnabled(isEnabled);
818 if (!isEnabled) {
819 return;
820 }
821
822 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY,
823 "send accessibility componentType:%{public}s event:%{public}d accessibilityId:%{public}" PRId64,
824 eventInfo.GetComponentType().c_str(), eventInfo.GetEventType(), eventInfo.GetAccessibilityId());
825 client->SendEvent(eventInfo);
826 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
827 return true;
828 }
829 } // namespace OHOS::Ace::Framework
830