1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "js_third_provider_interaction_operation.h"
17
18 #include "accessibility_system_ability_client.h"
19
20 #include "core/pipeline_ng/pipeline_context.h"
21 #include "js_third_provider_interaction_operation_utils.h"
22
23 namespace OHOS::Ace::Framework {
24 namespace {
LogAccessibilityElementInfo(const std::string & tag,const Accessibility::AccessibilityElementInfo & info)25 void LogAccessibilityElementInfo(
26 const std::string& tag, const Accessibility::AccessibilityElementInfo& info)
27 {
28 std::string printStr;
29 printStr.append(tag).append(", info: [");
30 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
31 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
32 printStr.append(", childCount: ").append(std::to_string(info.GetChildCount()));
33 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
34 printStr.append(", parentNodeId: ").append(std::to_string(info.GetParentNodeId()));
35 printStr.append(", checkable: ").append(info.IsCheckable() ? "true" : "false");
36 printStr.append(", checked: ").append(info.IsChecked() ? "true" : "false");
37 printStr.append(", focusable: ").append(info.IsFocusable() ? "true" : "false");
38 printStr.append(", focused: ").append(info.IsFocused() ? "true" : "false");
39 printStr.append(", visible: ").append(info.IsVisible() ? "true" : "false");
40 printStr.append(", hasAccessibilityFocus: ")
41 .append(info.HasAccessibilityFocus() ? "true" : "false");
42 printStr.append(", selected: ").append(info.IsSelected() ? "true" : "false");
43 printStr.append(", clickable: ").append(info.IsClickable() ? "true" : "false");
44 printStr.append(", longClickable: ").append(info.IsLongClickable() ? "true" : "false");
45 printStr.append(", enabled: ").append(info.IsEnabled() ? "true" : "false");
46 printStr.append(", componentType: ").append(info.GetComponentType());
47 printStr.append(", content: ").append(info.GetContent());
48 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
49 printStr.append(", triggerAction: ")
50 .append(std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
51 printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
52 printStr.append(", childTreeId: ").append(std::to_string(info.GetChildTreeId()));
53 printStr.append(", belongTreeId: ").append(std::to_string(info.GetBelongTreeId()));
54 printStr.append(", parentWindowId: ").append(std::to_string(info.GetParentWindowId()));
55 printStr.append(", accessibilityGroup: ").append(info.GetAccessibilityGroup() ? "true" : "false");
56 std::string actionStr;
57 actionStr.append("{");
58 for (const auto& action : info.GetActionList()) {
59 actionStr.append(std::to_string(
60 static_cast<int32_t>(action.GetActionType()))).append(" ");
61 }
62 actionStr.append("}");
63 printStr.append(", action: ").append(actionStr);
64 printStr.append("]");
65 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
66 }
67
LogAccessibilityEventInfo(const std::string & tag,const Accessibility::AccessibilityEventInfo & info)68 void LogAccessibilityEventInfo(
69 const std::string& tag, const Accessibility::AccessibilityEventInfo& info)
70 {
71 std::string printStr;
72 printStr.append(tag).append(", info: [");
73 printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
74 printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
75 printStr.append(", viewId: ").append(std::to_string(info.GetViewId()));
76 printStr.append(", componentType: ").append(info.GetComponentType());
77 printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
78 printStr.append(", eventType: ").append(
79 std::to_string(static_cast<int32_t>(info.GetEventType())));
80 printStr.append(", triggerAction: ").append(
81 std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
82 printStr.append(", requestFocusElementId: ").append(
83 std::to_string(info.GetRequestFocusElementId()));
84 printStr.append("]");
85 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
86 }
FillNodeConfig(const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)87 void FillNodeConfig(
88 const NodeConfig& config, Accessibility::AccessibilityElementInfo& info)
89 {
90 OHOS::Accessibility::Rect oldRect = info.GetRectInScreen();
91 OHOS::Accessibility::Rect newRect = OHOS::Accessibility::Rect(
92 oldRect.GetLeftTopXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
93 oldRect.GetLeftTopYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()),
94 oldRect.GetRightBottomXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
95 oldRect.GetRightBottomYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()));
96 info.SetRectInScreen(newRect);
97 info.SetPageId(config.pageId);
98 info.SetWindowId(config.windowId);
99 info.SetBelongTreeId(config.belongTreeId);
100 info.SetChildTreeIdAndWinId(config.childTreeId, config.childWindowId);
101 info.SetParentWindowId(config.parentWindowId);
102 info.SetBundleName(config.bundleName);
103 info.SetInspectorKey(config.inspectorKey);
104 int64_t splitElementId = info.GetAccessibilityId();
105 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(config.belongTreeId, splitElementId);
106 info.SetAccessibilityId(splitElementId);
107 }
108
CopyNativeInfoToAccessibilityElementInfo(const ArkUI_AccessibilityElementInfo & elementInfo,const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)109 void CopyNativeInfoToAccessibilityElementInfo(
110 const ArkUI_AccessibilityElementInfo& elementInfo,
111 const NodeConfig& config,
112 Accessibility::AccessibilityElementInfo& info)
113 {
114 TransformAccessbilityElementInfo(elementInfo, info);
115 FillNodeConfig(config, info);
116 }
117
CopyNativeInfosToAccessibilityElementInfos(const std::vector<ArkUI_AccessibilityElementInfo> & nativeInfos,const NodeConfig & config,std::list<Accessibility::AccessibilityElementInfo> & infos)118 void CopyNativeInfosToAccessibilityElementInfos(
119 const std::vector<ArkUI_AccessibilityElementInfo>& nativeInfos,
120 const NodeConfig& config,
121 std::list<Accessibility::AccessibilityElementInfo>& infos)
122 {
123 for (const auto& nativeInfo : nativeInfos) {
124 Accessibility::AccessibilityElementInfo info;
125 TransformAccessbilityElementInfo(nativeInfo, info);
126 FillNodeConfig(config, info);
127 infos.push_back(info);
128 }
129 }
130
CheckEventIgnoreHostOffset(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent)131 bool CheckEventIgnoreHostOffset(
132 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent)
133 {
134 auto eventType = nativeAccessibilityEvent.GetEventType();
135 bool ignoreHostOffset = false;
136 switch (eventType) {
137 case ArkUI_AccessibilityEventType::
138 ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
139 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by focus update");
140 ignoreHostOffset = true;
141 break;
142 default:
143 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by other eventType");
144 }
145 return ignoreHostOffset;
146 }
147
RemoveKeysForClickAction(int32_t action,std::map<std::string,std::string> & args)148 void RemoveKeysForClickAction(
149 int32_t action,
150 std::map<std::string, std::string>& args)
151 {
152 auto aceAction = static_cast<ActionType>(action);
153 if (aceAction != ActionType::ACCESSIBILITY_ACTION_CLICK) {
154 return;
155 }
156
157 if (auto it = args.find(ACTION_ARGU_CLICK_ENHANCE_DATA); it != args.end()) {
158 args.erase(it);
159 }
160 if (auto it = args.find(ACTION_ARGU_CLICK_TIMESTAMP); it != args.end()) {
161 args.erase(it);
162 }
163 }
164 } // namespace
165
JsThirdProviderInteractionOperation(const WeakPtr<AccessibilityProvider> & accessibilityProvider,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager,WeakPtr<NG::FrameNode> host)166 JsThirdProviderInteractionOperation::JsThirdProviderInteractionOperation(
167 const WeakPtr<AccessibilityProvider>& accessibilityProvider,
168 const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager,
169 WeakPtr<NG::FrameNode> host)
170 : accessibilityProvider_(accessibilityProvider),
171 jsAccessibilityManager_(jsAccessibilityManager),
172 host_(host)
173 {
174 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Create");
175 }
176
~JsThirdProviderInteractionOperation()177 JsThirdProviderInteractionOperation::~JsThirdProviderInteractionOperation()
178 {
179 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Destory");
180 }
181
Initialize()182 void JsThirdProviderInteractionOperation::Initialize()
183 {
184 auto provider = accessibilityProvider_.Upgrade();
185 CHECK_NULL_VOID(provider);
186 }
187
SearchElementInfoByAccessibilityId(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t mode)188 RetError JsThirdProviderInteractionOperation::SearchElementInfoByAccessibilityId(
189 const int64_t elementId, const int32_t requestId,
190 Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode)
191 {
192 uint32_t realMode = mode;
193 if (realMode & static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED)) {
194 realMode &= ~static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED);
195 realMode |= static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN);
196 }
197 // 1. Get real elementId
198 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
199 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
200 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
201 elementId, splitElementId, splitTreeId);
202
203 // 2. FindAccessibilityNodeInfosById by provider
204 std::list<Accessibility::AccessibilityElementInfo> infos;
205 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
206 splitElementId, realMode, requestId, infos);
207 if (!ret) {
208 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
209 "SearchElementInfoByAccessibilityId failed.");
210 infos.clear();
211 }
212
213 // 3. Return result
214 SetSearchElementInfoByAccessibilityIdResult(callback, std::move(infos), requestId);
215 return RET_OK;
216 }
217
FindAccessibilityNodeInfosByIdFromProvider(const int64_t splitElementId,const int32_t mode,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos,bool ignoreHostOffset)218 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByIdFromProvider(
219 const int64_t splitElementId, const int32_t mode, const int32_t requestId,
220 std::list<Accessibility::AccessibilityElementInfo>& infos, bool ignoreHostOffset)
221 {
222 auto provider = accessibilityProvider_.Upgrade();
223 CHECK_NULL_RETURN(provider, false);
224 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
225 int32_t code = provider->FindAccessibilityNodeInfosById(
226 splitElementId, mode, requestId, nativeInfos);
227 if (code != 0) {
228 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
229 "FindAccessibilityNodeInfosByIdFromProvider failed: %{public}d", code);
230 return false;
231 }
232
233 NodeConfig config;
234 config.ignoreHostOffset = ignoreHostOffset;
235 GetNodeConfig(config);
236 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
237 return !infos.empty();
238 }
239
SetSearchElementInfoByAccessibilityIdResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)240 void JsThirdProviderInteractionOperation::SetSearchElementInfoByAccessibilityIdResult(
241 AccessibilityElementOperatorCallback& callback,
242 std::list<AccessibilityElementInfo>&& infos,
243 const int32_t requestId)
244 {
245 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
246 CHECK_NULL_VOID(jsAccessibilityManager);
247 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
248 CHECK_NULL_VOID(context);
249 CHECK_NULL_VOID(context->GetTaskExecutor());
250 context->GetTaskExecutor()->PostTask(
251 [jsMgr = jsAccessibilityManager_,
252 infos = std::move(infos), &callback, requestId] () mutable {
253 auto jsAccessibilityManager = jsMgr.Upgrade();
254 CHECK_NULL_VOID(jsAccessibilityManager);
255 if (!jsAccessibilityManager->IsRegister()) {
256 return;
257 }
258 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
259 callback.SetSearchElementInfoByAccessibilityIdResult(infos, requestId);
260 }, TaskExecutor::TaskType::BACKGROUND, "SearchElementInfoByAccessibilityId");
261 }
262
SearchElementInfoBySpecificProperty(const int64_t elementId,const SpecificPropertyParam & param,const int32_t requestId,AccessibilityElementOperatorCallback & callback)263 void JsThirdProviderInteractionOperation::SearchElementInfoBySpecificProperty(const int64_t elementId,
264 const SpecificPropertyParam ¶m, const int32_t requestId,
265 AccessibilityElementOperatorCallback &callback)
266 {
267 std::list<AccessibilityElementInfo> infos;
268 std::list<AccessibilityElementInfo> treeInfos;
269 callback.SetSearchElementInfoBySpecificPropertyResult(infos, treeInfos, requestId);
270 }
271
SearchElementInfosByText(const int64_t elementId,const std::string & text,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)272 void JsThirdProviderInteractionOperation::SearchElementInfosByText(
273 const int64_t elementId, const std::string& text, const int32_t requestId,
274 Accessibility::AccessibilityElementOperatorCallback& callback)
275 {
276 // 1. Get real elementId
277 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
278 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
279 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
280 elementId, splitElementId, splitTreeId);
281
282 // 2. FindAccessibilityNodeInfosByText from provider
283 std::list<Accessibility::AccessibilityElementInfo> infos;
284 bool ret = FindAccessibilityNodeInfosByTextFromProvider(
285 splitElementId, text, requestId, infos);
286 if (!ret) {
287 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
288 "SearchElementInfosByText failed.");
289 infos.clear();
290 }
291
292 // 3. Return result
293 SetSearchElementInfoByTextResult(callback, std::move(infos), requestId);
294 }
295
SearchDefaultFocusByWindowId(const int32_t windowId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t pageId)296 void JsThirdProviderInteractionOperation::SearchDefaultFocusByWindowId(
297 const int32_t windowId, const int32_t requestId,
298 Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t pageId)
299 {
300 }
301
FindAccessibilityNodeInfosByTextFromProvider(const int64_t splitElementId,const std::string & text,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos)302 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByTextFromProvider(
303 const int64_t splitElementId, const std::string& text, const int32_t requestId,
304 std::list<Accessibility::AccessibilityElementInfo>& infos)
305 {
306 auto provider = accessibilityProvider_.Upgrade();
307 CHECK_NULL_RETURN(provider, false);
308 std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
309 int32_t code = provider->FindAccessibilityNodeInfosByText(
310 splitElementId, text, requestId, nativeInfos);
311 if (code != 0) {
312 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
313 "FindAccessibilityNodeInfosByTextFromProvider failed: %{public}d", code);
314 return false;
315 }
316
317 NodeConfig config;
318 GetNodeConfig(config);
319 CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
320 return true;
321 }
322
SetSearchElementInfoByTextResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)323 void JsThirdProviderInteractionOperation::SetSearchElementInfoByTextResult(
324 AccessibilityElementOperatorCallback& callback,
325 std::list<AccessibilityElementInfo>&& infos,
326 const int32_t requestId)
327 {
328 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
329 CHECK_NULL_VOID(jsAccessibilityManager);
330 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
331 CHECK_NULL_VOID(context);
332 CHECK_NULL_VOID(context->GetTaskExecutor());
333 context->GetTaskExecutor()->PostTask(
334 [jsMgr = jsAccessibilityManager_,
335 infos = std::move(infos), &callback, requestId] () mutable {
336 auto jsAccessibilityManager = jsMgr.Upgrade();
337 CHECK_NULL_VOID(jsAccessibilityManager);
338 if (!jsAccessibilityManager->IsRegister()) {
339 return;
340 }
341 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
342 callback.SetSearchElementInfoByTextResult(infos, requestId);
343 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
344 }
345
FindFocusedElementInfo(const int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)346 void JsThirdProviderInteractionOperation::FindFocusedElementInfo(
347 const int64_t elementId, const int32_t focusType, const int32_t requestId,
348 Accessibility::AccessibilityElementOperatorCallback& callback)
349 {
350 // 1. Get real elementId
351 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
352 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
353 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
354 elementId, splitElementId, splitTreeId);
355
356 // 2. FindFocusedAccessibilityNode from provider
357 Accessibility::AccessibilityElementInfo info;
358 bool ret = FindFocusedElementInfoFromProvider(
359 splitElementId, focusType, requestId, info);
360 if (!ret) {
361 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
362 "SearchElementInfosByText failed.");
363 info.SetValidElement(false);
364 }
365
366 // 3. Return result
367 SetFindFocusedElementInfoResult(callback, info, requestId);
368 }
369
FindFocusedElementInfoFromProvider(int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)370 bool JsThirdProviderInteractionOperation::FindFocusedElementInfoFromProvider(
371 int64_t elementId, const int32_t focusType, const int32_t requestId,
372 Accessibility::AccessibilityElementInfo& info)
373 {
374 auto provider = accessibilityProvider_.Upgrade();
375 CHECK_NULL_RETURN(provider, false);
376 ArkUI_AccessibilityElementInfo nativeInfo;
377 int32_t code = provider->FindFocusedAccessibilityNode(
378 elementId, focusType, requestId, nativeInfo);
379 if (code != 0) {
380 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
381 "FindFocusedElementInfoFromProvider failed: %{public}d", code);
382 return false;
383 }
384
385 NodeConfig config;
386 GetNodeConfig(config);
387 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
388 return true;
389 }
390
SetFindFocusedElementInfoResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)391 void JsThirdProviderInteractionOperation::SetFindFocusedElementInfoResult(
392 AccessibilityElementOperatorCallback& callback,
393 AccessibilityElementInfo& info,
394 const int32_t requestId)
395 {
396 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
397 CHECK_NULL_VOID(jsAccessibilityManager);
398 jsAccessibilityManager->UpdateElementInfoTreeId(info);
399 callback.SetFindFocusedElementInfoResult(info, requestId);
400 }
401
FocusMoveSearch(const int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)402 void JsThirdProviderInteractionOperation::FocusMoveSearch(
403 const int64_t elementId, const int32_t direction, const int32_t requestId,
404 Accessibility::AccessibilityElementOperatorCallback& callback)
405 {
406 // 1. Get real elementId
407 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
408 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
409 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
410 elementId, splitElementId, splitTreeId);
411
412 // 2. FindNextFocusAccessibilityNode from provider
413 Accessibility::AccessibilityElementInfo info;
414 bool ret = FocusMoveSearchProvider(
415 splitElementId, direction, requestId, info);
416 // 3. Return result
417 if (!ret) {
418 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
419 CHECK_NULL_VOID(jsAccessibilityManager);
420 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
421 CHECK_NULL_VOID(context);
422 CHECK_NULL_VOID(context->GetTaskExecutor());
423 context->GetTaskExecutor()->PostTask(
424 [jsMgr = jsAccessibilityManager_, weakHost = host_, &callback, requestId] () {
425 auto jsAccessibilityManager = jsMgr.Upgrade();
426 if ((!jsAccessibilityManager) || (!jsAccessibilityManager->IsRegister())) {
427 AccessibilityElementInfo nodeInfo;
428 nodeInfo.SetValidElement(false);
429 callback.SetFocusMoveSearchResult(nodeInfo, requestId);
430 return;
431 }
432 jsAccessibilityManager->SetFocusMoveResultWithNode(weakHost, callback, requestId);
433 }, TaskExecutor::TaskType::UI, "AccessibilityThirdPartyFocusMoveSearchFail");
434 } else {
435 SetFocusMoveSearchResult(callback, info, requestId);
436 }
437 }
438
FocusMoveSearchProvider(int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)439 bool JsThirdProviderInteractionOperation::FocusMoveSearchProvider(
440 int64_t elementId, const int32_t direction, const int32_t requestId,
441 Accessibility::AccessibilityElementInfo& info)
442 {
443 auto provider = accessibilityProvider_.Upgrade();
444 CHECK_NULL_RETURN(provider, false);
445 ArkUI_AccessibilityElementInfo nativeInfo;
446 int32_t code = provider->FindNextFocusAccessibilityNode(
447 elementId, direction, requestId, nativeInfo);
448 if (code != 0) {
449 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
450 "FocusMoveSearchProvider failed: %{public}d", code);
451 return false;
452 }
453
454 NodeConfig config;
455 GetNodeConfig(config);
456 CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
457 return true;
458 }
459
SetFocusMoveSearchResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)460 void JsThirdProviderInteractionOperation::SetFocusMoveSearchResult(
461 AccessibilityElementOperatorCallback& callback,
462 AccessibilityElementInfo& info,
463 const int32_t requestId)
464 {
465 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
466 CHECK_NULL_VOID(jsAccessibilityManager);
467 jsAccessibilityManager->UpdateElementInfoTreeId(info);
468 callback.SetFocusMoveSearchResult(info, requestId);
469 }
470
HandleActionWhenFindNodeFail(const int32_t action)471 void JsThirdProviderInteractionOperation::HandleActionWhenFindNodeFail(const int32_t action)
472 {
473 if (action == static_cast<int32_t>(
474 Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
475 ClearDrawBound();
476 }
477 }
478
ExecuteAction(const int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)479 void JsThirdProviderInteractionOperation::ExecuteAction(
480 const int64_t elementId, const int32_t action,
481 const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
482 Accessibility::AccessibilityElementOperatorCallback& callback)
483 {
484 // 1. Get real elementId
485 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
486 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
487 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
488 elementId, splitElementId, splitTreeId);
489
490 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAction elementId: %{public}" PRId64 ","
491 " action: %{public}d, requestId: %{public}d, splitElementId: %{public}" PRId64 ","
492 " splitTreeId: %{public}d",
493 elementId, action, requestId, splitElementId, splitTreeId);
494 // 2. FindNextFocusAccessibilityNode from provider
495 std::list<Accessibility::AccessibilityElementInfo> infos;
496 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
497 splitElementId, 0, requestId, infos, true); // for drawbound, no need fix host offset
498 if (!ret) {
499 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "Find info failed when ExecuteAction.");
500 SetExecuteActionResult(callback, false, requestId);
501 HandleActionWhenFindNodeFail(action);
502 return;
503 }
504
505 // 3. DrawBound
506 ExecuteActionForThird(splitElementId, infos.front(), action);
507
508 //4. ExecuteAccessibilityAction To provider
509 ret = ExecuteActionFromProvider(splitElementId, action, actionArguments, requestId);
510 if (!ret) {
511 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAccessibilityAction failed.");
512 }
513
514 // 5. Return result
515 SetExecuteActionResult(callback, ret, requestId);
516 }
517
ExecuteActionFromProvider(int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId)518 bool JsThirdProviderInteractionOperation::ExecuteActionFromProvider(
519 int64_t elementId, const int32_t action,
520 const std::map<std::string, std::string>& actionArguments, const int32_t requestId)
521 {
522 auto actionFilteredArguments = actionArguments;
523 RemoveKeysForClickAction(action, actionFilteredArguments);
524
525 auto provider = accessibilityProvider_.Upgrade();
526 CHECK_NULL_RETURN(provider, false);
527 int32_t code = provider->ExecuteAccessibilityAction(
528 elementId, action, requestId, actionFilteredArguments);
529 if (code != 0) {
530 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
531 "ExecuteActionFromProvider failed: %{public}d", code);
532 return false;
533 }
534
535 return true;
536 }
537
ExecuteActionForThird(int64_t elementId,const AccessibilityElementInfo & nodeInfo,const int32_t action)538 bool JsThirdProviderInteractionOperation::ExecuteActionForThird(
539 int64_t elementId, const AccessibilityElementInfo& nodeInfo, const int32_t action)
540 {
541 auto hostNode = GetHost();
542 CHECK_NULL_RETURN(hostNode, false);
543 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
544 CHECK_NULL_RETURN(jsAccessibilityManager, false);
545 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
546 CHECK_NULL_RETURN(context, false);
547 auto ngPipeline = AceType::DynamicCast<NG::PipelineContext>(context);
548 CHECK_NULL_RETURN(ngPipeline, false);
549 if (action == static_cast<int32_t>(
550 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS)) {
551 jsAccessibilityManager->ActThirdAccessibilityFocus(
552 elementId, nodeInfo, hostNode, ngPipeline, false);
553 } else if (action == static_cast<int32_t>(
554 Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
555 jsAccessibilityManager->ActThirdAccessibilityFocus(
556 elementId, nodeInfo, hostNode, ngPipeline, true);
557 }
558 return true;
559 }
560
SetExecuteActionResult(AccessibilityElementOperatorCallback & callback,const bool succeeded,const int32_t requestId)561 void JsThirdProviderInteractionOperation::SetExecuteActionResult(
562 AccessibilityElementOperatorCallback& callback,
563 const bool succeeded, const int32_t requestId)
564 {
565 callback.SetExecuteActionResult(succeeded, requestId);
566 }
567
ClearFocus()568 void JsThirdProviderInteractionOperation::ClearFocus()
569 {
570 // 1. Clear focus from provider
571 ClearFocusFromProvider();
572 // 2. Clear DrawBound
573 ClearDrawBound();
574 }
575
ClearFocusFromProvider()576 bool JsThirdProviderInteractionOperation::ClearFocusFromProvider()
577 {
578 auto provider = accessibilityProvider_.Upgrade();
579 CHECK_NULL_RETURN(provider, false);
580 int32_t code = provider->ClearFocusedAccessibilityNode();
581 if (code != 0) {
582 TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
583 "ExecuteActionFromProvider failed: %{public}d", code);
584 return false;
585 }
586 return true;
587 }
588
ClearDrawBound()589 bool JsThirdProviderInteractionOperation::ClearDrawBound()
590 {
591 auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
592 CHECK_NULL_RETURN(jsAccessibilityManager, false);
593 auto hostNode = GetHost();
594 CHECK_NULL_RETURN(hostNode, false);
595 return jsAccessibilityManager->ClearThirdAccessibilityFocus(hostNode);
596 }
597
OutsideTouch()598 void JsThirdProviderInteractionOperation::OutsideTouch()
599 {}
600
GetCursorPosition(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)601 void JsThirdProviderInteractionOperation::GetCursorPosition(
602 const int64_t elementId, const int32_t requestId,
603 Accessibility::AccessibilityElementOperatorCallback &callback)
604 {
605 auto provider = accessibilityProvider_.Upgrade();
606 CHECK_NULL_VOID(provider);
607 int32_t cursorPosition = -1;
608 int32_t code = provider->GetAccessibilityNodeCursorPosition(
609 elementId, requestId, cursorPosition);
610 if (code != 0) {
611 callback.SetCursorPositionResult(-1, requestId);
612 return;
613 }
614
615 callback.SetCursorPositionResult(cursorPosition, requestId);
616 }
617
GetHostRectTranslateInfo(NodeConfig & config)618 void JsThirdProviderInteractionOperation::GetHostRectTranslateInfo(NodeConfig& config)
619 {
620 auto host = host_.Upgrade();
621 if ((!host) || config.ignoreHostOffset) {
622 config.offset = NG::OffsetF(0, 0);
623 config.scaleX = 1.0f;
624 config.scaleY = 1.0f;
625 return;
626 }
627 auto rect = host->GetTransformRectRelativeToWindow();
628 NG::VectorF finalScale = host->GetTransformScaleRelativeToWindow();
629 auto pipeline = host->GetContextRefPtr();
630 if (pipeline) {
631 auto accessibilityManager = pipeline->GetAccessibilityManager();
632 if (accessibilityManager) {
633 auto windowInfo = accessibilityManager->GenerateWindowInfo(host, pipeline);
634 auto top = rect.Top() * windowInfo.scaleY + static_cast<int32_t>(windowInfo.top);
635 auto left = rect.Left() * windowInfo.scaleX + static_cast<int32_t>(windowInfo.left);
636 finalScale.x *= windowInfo.scaleX;
637 finalScale.y *= windowInfo.scaleY;
638 config.offset = NG::OffsetT(left, top);
639 } else {
640 auto windowRect = pipeline->GetDisplayWindowRectInfo();
641 auto top = rect.Top() + static_cast<int32_t>(windowRect.Top());
642 auto left = rect.Left() + static_cast<int32_t>(windowRect.Left());
643 config.offset = NG::OffsetT(left, top);
644 }
645 }
646
647 config.scaleX = finalScale.x;
648 config.scaleY = finalScale.y;
649 }
650
GetNodeConfig(NodeConfig & config)651 void JsThirdProviderInteractionOperation::GetNodeConfig(NodeConfig& config)
652 {
653 auto host = host_.Upgrade();
654 CHECK_NULL_VOID(host);
655 auto jsAccessibilityManager = GetHandler().Upgrade();
656 CHECK_NULL_VOID(jsAccessibilityManager);
657 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
658 CHECK_NULL_VOID(context);
659 config.pageId = host->GetPageId();
660 config.windowId = static_cast<int32_t>(context->GetRealHostWindowId());
661 config.belongTreeId = belongTreeId_;
662 config.parentWindowId = static_cast<int32_t>(context->GetRealHostWindowId());
663 config.bundleName = AceApplicationInfo::GetInstance().GetPackageName();
664
665 GetHostRectTranslateInfo(config);
666 }
667
SetChildTreeIdAndWinId(const int64_t nodeId,const int32_t treeId,const int32_t childWindowId)668 void JsThirdProviderInteractionOperation::SetChildTreeIdAndWinId(
669 const int64_t nodeId, const int32_t treeId, const int32_t childWindowId)
670 {
671 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetChildTreeIdAndWinId, node: %{public}" PRId64 ","
672 "treeId: %{public}d, childWindowId: %{public}d", nodeId, treeId, childWindowId);
673 }
674
SetBelongTreeId(const int32_t treeId)675 void JsThirdProviderInteractionOperation::SetBelongTreeId(const int32_t treeId)
676 {
677 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "SetBelongTreeId treeId: %{public}d", treeId);
678 belongTreeId_ = treeId;
679 }
680
SendAccessibilityAsyncEventForThird(int64_t thirdElementId,Accessibility::EventType eventType)681 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEventForThird(
682 int64_t thirdElementId,
683 Accessibility::EventType eventType)
684 {
685 // 1. generate event
686 Accessibility::AccessibilityEventInfo event;
687 event.SetTimeStamp(GetMicroTickCount());
688 event.SetWindowChangeTypes(
689 Accessibility::WindowUpdateType::WINDOW_UPDATE_INVALID);
690 event.SetWindowContentChangeTypes(
691 Accessibility::WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_INVALID);
692 event.SetSource(thirdElementId);
693 event.SetEventType(eventType);
694 event.SetBundleName(AceApplicationInfo::GetInstance().GetPackageName());
695
696 // 2. get element from third
697 // cut tree id
698 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
699 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
700 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
701 thirdElementId, splitElementId, splitTreeId);
702 std::list<Accessibility::AccessibilityElementInfo> infos;
703 bool ret = FindAccessibilityNodeInfosByIdFromProvider(
704 splitElementId, 0, 0, infos);
705 if ((!ret) || (infos.size() == 0)) {
706 return -1;
707 }
708
709 auto jsAccessibilityManager = GetHandler().Upgrade();
710 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
711 jsAccessibilityManager->UpdateElementInfosTreeId(infos);
712 event.SetElementInfo(infos.front());
713
714 // 3. change event info by host info
715 GetAccessibilityEventInfoFromNativeEvent(event, false);
716
717 // 4. SendEvent
718 auto host = host_.Upgrade();
719 CHECK_NULL_RETURN(host, -1);
720 SendAccessibilitySyncEventToService(event, nullptr);
721 return 0;
722 }
723
HandleNativeFocusUpdate(int64_t elementId,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)724 bool JsThirdProviderInteractionOperation::HandleNativeFocusUpdate(
725 int64_t elementId,
726 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
727 {
728 ExecuteActionForThird(
729 elementId,
730 accessibilityEventInfo.GetElementInfo(),
731 static_cast<int32_t>(
732 Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS));
733 return false;
734 }
735
HandleEventByFramework(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)736 bool JsThirdProviderInteractionOperation::HandleEventByFramework(
737 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
738 Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
739 {
740 auto eventType = nativeAccessibilityEvent.GetEventType();
741 bool needSendEvent = true;
742 switch (eventType) {
743 case ArkUI_AccessibilityEventType::
744 ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
745 if (nativeAccessibilityEvent.GetElementInfo()) {
746 needSendEvent = HandleNativeFocusUpdate(
747 nativeAccessibilityEvent.GetElementInfo()->GetElementId(),
748 accessibilityEventInfo);
749 }
750 break;
751 default:
752 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Unsupported eventType");
753 }
754 return needSendEvent;
755 }
756
SendAccessibilityAsyncEvent(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,void (* callback)(int32_t errorCode))757 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEvent(
758 const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
759 void (*callback)(int32_t errorCode))
760 {
761 bool ignoreHostOffset = CheckEventIgnoreHostOffset(nativeAccessibilityEvent);
762 // 1. Get OHOS::Accessibility::AccessibilityEventInfo
763 OHOS::Accessibility::AccessibilityEventInfo accessibilityEventInfo;
764 GetAccessibilityEventInfoFromNativeEvent(
765 nativeAccessibilityEvent, accessibilityEventInfo, ignoreHostOffset);
766
767 // 2. handleEvent by frame work
768 bool needSendEvent = HandleEventByFramework(
769 nativeAccessibilityEvent,
770 accessibilityEventInfo);
771 // 3. SendEvent
772 if (needSendEvent) {
773 auto jsAccessibilityManager = GetHandler().Upgrade();
774 CHECK_NULL_RETURN(jsAccessibilityManager, -1);
775 auto host = host_.Upgrade();
776 CHECK_NULL_RETURN(host, -1);
777 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Inner SendAccessibilityAsyncEvent");
778 SendAccessibilitySyncEventToService(accessibilityEventInfo, callback);
779 }
780 callback(0);
781 return 0;
782 }
783
GetAccessibilityEventInfoFromNativeEvent(OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)784 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
785 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
786 bool ignoreHostOffset)
787 {
788 // 1. Fill node config
789 NodeConfig config;
790 config.ignoreHostOffset = ignoreHostOffset;
791 GetNodeConfig(config);
792
793 // 1.1. Fill elementInfo config
794 auto elementInfo = accessibilityEventInfo.GetElementInfo();
795 FillNodeConfig(config, elementInfo);
796 int64_t elementId = elementInfo.GetAccessibilityId();
797 AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(belongTreeId_, elementId);
798 elementInfo.SetAccessibilityId(elementId);
799
800 // 1.2. Fill eventInfo config
801 accessibilityEventInfo.SetPageId(config.pageId);
802 accessibilityEventInfo.SetWindowId(config.windowId);
803 accessibilityEventInfo.SetSource(elementId);
804 accessibilityEventInfo.SetComponentType(elementInfo.GetComponentType());
805 accessibilityEventInfo.AddContent(elementInfo.GetContent());
806 accessibilityEventInfo.SetElementInfo(elementInfo);
807 LogAccessibilityElementInfo("sendEvent", elementInfo);
808 LogAccessibilityEventInfo("sendEvent", accessibilityEventInfo);
809 }
810
GetAccessibilityEventInfoFromNativeEvent(const ArkUI_AccessibilityEventInfo & nativeEventInfo,OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)811 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
812 const ArkUI_AccessibilityEventInfo& nativeEventInfo,
813 OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
814 bool ignoreHostOffset)
815 {
816 // 1. Transform native event to OHOS::Accessibility::AccessibilityEventInfo
817 TransformAccessbilityEventInfo(
818 nativeEventInfo, accessibilityEventInfo);
819
820 // 2. Transform Accessibility::AccessibilityEventInfo with host info
821 GetAccessibilityEventInfoFromNativeEvent(accessibilityEventInfo, ignoreHostOffset);
822 }
823
SendAccessibilitySyncEventToService(const OHOS::Accessibility::AccessibilityEventInfo & eventInfo,void (* callback)(int32_t errorCode))824 bool JsThirdProviderInteractionOperation::SendAccessibilitySyncEventToService(
825 const OHOS::Accessibility::AccessibilityEventInfo& eventInfo,
826 [[maybe_unused]] void (*callback)(int32_t errorCode))
827 {
828 auto jsAccessibilityManager = GetHandler().Upgrade();
829 CHECK_NULL_RETURN(jsAccessibilityManager, false);
830 auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
831 CHECK_NULL_RETURN(context, false);
832 CHECK_NULL_RETURN(context->GetTaskExecutor(), false);
833 context->GetTaskExecutor()->PostTask(
834 [jsMgr = jsAccessibilityManager_, eventInfo] () mutable {
835 auto jsAccessibilityManager = jsMgr.Upgrade();
836 if (jsAccessibilityManager == nullptr) {
837 return;
838 }
839
840 if (!jsAccessibilityManager->IsRegister()) {
841 return;
842 }
843
844 auto client = AccessibilitySystemAbilityClient::GetInstance();
845 CHECK_NULL_VOID(client);
846 bool isEnabled = false;
847 client->IsEnabled(isEnabled);
848 if (!isEnabled) {
849 return;
850 }
851
852 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY,
853 "send accessibility componentType:%{public}s event:%{public}d accessibilityId:%{public}" PRId64,
854 eventInfo.GetComponentType().c_str(), eventInfo.GetEventType(), eventInfo.GetAccessibilityId());
855 client->SendEvent(eventInfo);
856 }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
857 return true;
858 }
859 } // namespace OHOS::Ace::Framework
860