• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "js_third_provider_interaction_operation.h"
17 
18 #include "accessibility_system_ability_client.h"
19 
20 #include "core/pipeline_ng/pipeline_context.h"
21 #include "js_third_provider_interaction_operation_utils.h"
22 
23 namespace OHOS::Ace::Framework {
24 namespace {
LogAccessibilityElementInfo(const std::string & tag,const Accessibility::AccessibilityElementInfo & info)25 void LogAccessibilityElementInfo(
26     const std::string& tag, const Accessibility::AccessibilityElementInfo& info)
27 {
28     std::string printStr;
29     printStr.append(tag).append(", info: [");
30     printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
31     printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
32     printStr.append(", childCount: ").append(std::to_string(info.GetChildCount()));
33     printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
34     printStr.append(", parentNodeId: ").append(std::to_string(info.GetParentNodeId()));
35     printStr.append(", checkable: ").append(info.IsCheckable() ? "true" : "false");
36     printStr.append(", checked: ").append(info.IsChecked() ? "true" : "false");
37     printStr.append(", focusable: ").append(info.IsFocusable() ? "true" : "false");
38     printStr.append(", focused: ").append(info.IsFocused() ? "true" : "false");
39     printStr.append(", visible: ").append(info.IsVisible() ? "true" : "false");
40     printStr.append(", hasAccessibilityFocus: ")
41         .append(info.HasAccessibilityFocus() ? "true" : "false");
42     printStr.append(", selected: ").append(info.IsSelected() ? "true" : "false");
43     printStr.append(", clickable: ").append(info.IsClickable() ? "true" : "false");
44     printStr.append(", longClickable: ").append(info.IsLongClickable() ? "true" : "false");
45     printStr.append(", enabled: ").append(info.IsEnabled() ? "true" : "false");
46     printStr.append(", componentType: ").append(info.GetComponentType());
47     printStr.append(", content: ").append(info.GetContent());
48     printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
49     printStr.append(", triggerAction: ")
50         .append(std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
51     printStr.append(", accessibilityText: ").append(info.GetAccessibilityText());
52     printStr.append(", childTreeId: ").append(std::to_string(info.GetChildTreeId()));
53     printStr.append(", belongTreeId: ").append(std::to_string(info.GetBelongTreeId()));
54     printStr.append(", parentWindowId: ").append(std::to_string(info.GetParentWindowId()));
55     printStr.append(", accessibilityGroup: ").append(info.GetAccessibilityGroup() ? "true" : "false");
56     std::string actionStr;
57     actionStr.append("{");
58     for (const auto& action : info.GetActionList()) {
59         actionStr.append(std::to_string(
60             static_cast<int32_t>(action.GetActionType()))).append(" ");
61     }
62     actionStr.append("}");
63     printStr.append(", action: ").append(actionStr);
64     printStr.append("]");
65     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
66 }
67 
LogAccessibilityEventInfo(const std::string & tag,const Accessibility::AccessibilityEventInfo & info)68 void LogAccessibilityEventInfo(
69     const std::string& tag, const Accessibility::AccessibilityEventInfo& info)
70 {
71     std::string printStr;
72     printStr.append(tag).append(", info: [");
73     printStr.append("accessibilityId: ").append(std::to_string(info.GetAccessibilityId()));
74     printStr.append(", windowId: ").append(std::to_string(info.GetWindowId()));
75     printStr.append(", viewId: ").append(std::to_string(info.GetViewId()));
76     printStr.append(", componentType: ").append(info.GetComponentType());
77     printStr.append(", pageId: ").append(std::to_string(info.GetPageId()));
78     printStr.append(", eventType: ").append(
79         std::to_string(static_cast<int32_t>(info.GetEventType())));
80     printStr.append(", triggerAction: ").append(
81         std::to_string(static_cast<int32_t>(info.GetTriggerAction())));
82     printStr.append(", requestFocusElementId: ").append(
83         std::to_string(info.GetRequestFocusElementId()));
84     printStr.append("]");
85     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "%{public}s", printStr.c_str());
86 }
FillNodeConfig(const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)87 void FillNodeConfig(
88     const NodeConfig& config, Accessibility::AccessibilityElementInfo& info)
89 {
90     OHOS::Accessibility::Rect oldRect = info.GetRectInScreen();
91     OHOS::Accessibility::Rect newRect = OHOS::Accessibility::Rect(
92         oldRect.GetLeftTopXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
93         oldRect.GetLeftTopYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()),
94         oldRect.GetRightBottomXScreenPostion() * config.scaleX + static_cast<int32_t>(config.offset.GetX()),
95         oldRect.GetRightBottomYScreenPostion() * config.scaleY + static_cast<int32_t>(config.offset.GetY()));
96     info.SetRectInScreen(newRect);
97     info.SetPageId(config.pageId);
98     info.SetWindowId(config.windowId);
99     info.SetBelongTreeId(config.belongTreeId);
100     info.SetChildTreeIdAndWinId(config.childTreeId, config.childWindowId);
101     info.SetParentWindowId(config.parentWindowId);
102     info.SetBundleName(config.bundleName);
103     info.SetInspectorKey(config.inspectorKey);
104     int64_t splitElementId = info.GetAccessibilityId();
105     AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(config.belongTreeId, splitElementId);
106     info.SetAccessibilityId(splitElementId);
107 }
108 
CopyNativeInfoToAccessibilityElementInfo(const ArkUI_AccessibilityElementInfo & elementInfo,const NodeConfig & config,Accessibility::AccessibilityElementInfo & info)109 void CopyNativeInfoToAccessibilityElementInfo(
110     const ArkUI_AccessibilityElementInfo& elementInfo,
111     const NodeConfig& config,
112     Accessibility::AccessibilityElementInfo& info)
113 {
114     TransformAccessbilityElementInfo(elementInfo, info);
115     FillNodeConfig(config, info);
116 }
117 
CopyNativeInfosToAccessibilityElementInfos(const std::vector<ArkUI_AccessibilityElementInfo> & nativeInfos,const NodeConfig & config,std::list<Accessibility::AccessibilityElementInfo> & infos)118 void CopyNativeInfosToAccessibilityElementInfos(
119     const std::vector<ArkUI_AccessibilityElementInfo>& nativeInfos,
120     const NodeConfig& config,
121     std::list<Accessibility::AccessibilityElementInfo>& infos)
122 {
123     for (const auto& nativeInfo : nativeInfos) {
124         Accessibility::AccessibilityElementInfo info;
125         TransformAccessbilityElementInfo(nativeInfo, info);
126         FillNodeConfig(config, info);
127         infos.push_back(info);
128     }
129 }
130 
CheckEventIgnoreHostOffset(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent)131 bool CheckEventIgnoreHostOffset(
132     const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent)
133 {
134     auto eventType = nativeAccessibilityEvent.GetEventType();
135     bool ignoreHostOffset = false;
136     switch (eventType) {
137         case ArkUI_AccessibilityEventType::
138             ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
139             TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by focus update");
140             ignoreHostOffset = true;
141             break;
142         default:
143             TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "ignoreHostOffset keep false by other eventType");
144     }
145     return ignoreHostOffset;
146 }
147 
RemoveKeysForClickAction(int32_t action,std::map<std::string,std::string> & args)148 void RemoveKeysForClickAction(
149     int32_t action,
150     std::map<std::string, std::string>& args)
151 {
152     auto aceAction = static_cast<ActionType>(action);
153     if (aceAction != ActionType::ACCESSIBILITY_ACTION_CLICK) {
154         return;
155     }
156 
157     if (auto it = args.find(ACTION_ARGU_CLICK_ENHANCE_DATA); it != args.end()) {
158         args.erase(it);
159     }
160     if (auto it = args.find(ACTION_ARGU_CLICK_TIMESTAMP); it != args.end()) {
161         args.erase(it);
162     }
163 }
164 } // namespace
165 
JsThirdProviderInteractionOperation(const WeakPtr<AccessibilityProvider> & accessibilityProvider,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager,WeakPtr<NG::FrameNode> host)166 JsThirdProviderInteractionOperation::JsThirdProviderInteractionOperation(
167     const WeakPtr<AccessibilityProvider>& accessibilityProvider,
168     const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager,
169     WeakPtr<NG::FrameNode> host)
170     : accessibilityProvider_(accessibilityProvider),
171     jsAccessibilityManager_(jsAccessibilityManager),
172     host_(host)
173 {
174     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Create");
175 }
176 
~JsThirdProviderInteractionOperation()177 JsThirdProviderInteractionOperation::~JsThirdProviderInteractionOperation()
178 {
179     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "JsThirdProviderInteractionOperation Destory");
180 }
181 
Initialize()182 void JsThirdProviderInteractionOperation::Initialize()
183 {
184     auto provider = accessibilityProvider_.Upgrade();
185     CHECK_NULL_VOID(provider);
186 }
187 
SearchElementInfoByAccessibilityId(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t mode)188 RetError JsThirdProviderInteractionOperation::SearchElementInfoByAccessibilityId(
189     const int64_t elementId, const int32_t requestId,
190     Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t mode)
191 {
192     uint32_t realMode = mode;
193     if (realMode & static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED)) {
194         realMode &= ~static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN_REDUCED);
195         realMode |= static_cast<uint32_t>(PREFETCH_RECURSIVE_CHILDREN);
196     }
197     // 1. Get real elementId
198     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
199     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
200     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
201         elementId, splitElementId, splitTreeId);
202 
203     // 2. FindAccessibilityNodeInfosById by provider
204     std::list<Accessibility::AccessibilityElementInfo> infos;
205     bool ret = FindAccessibilityNodeInfosByIdFromProvider(
206         splitElementId, realMode, requestId, infos);
207     if (!ret) {
208         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
209             "SearchElementInfoByAccessibilityId failed.");
210         infos.clear();
211     }
212 
213     // 3. Return result
214     SetSearchElementInfoByAccessibilityIdResult(callback, std::move(infos), requestId);
215     return RET_OK;
216 }
217 
FindAccessibilityNodeInfosByIdFromProvider(const int64_t splitElementId,const int32_t mode,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos,bool ignoreHostOffset)218 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByIdFromProvider(
219     const int64_t splitElementId, const int32_t mode, const int32_t requestId,
220     std::list<Accessibility::AccessibilityElementInfo>& infos, bool ignoreHostOffset)
221 {
222     auto provider = accessibilityProvider_.Upgrade();
223     CHECK_NULL_RETURN(provider, false);
224     std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
225     int32_t code = provider->FindAccessibilityNodeInfosById(
226         splitElementId, mode, requestId, nativeInfos);
227     if (code != 0) {
228         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
229             "FindAccessibilityNodeInfosByIdFromProvider failed: %{public}d", code);
230         return false;
231     }
232 
233     NodeConfig config;
234     config.ignoreHostOffset = ignoreHostOffset;
235     GetNodeConfig(config);
236     CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
237     return !infos.empty();
238 }
239 
SetSearchElementInfoByAccessibilityIdResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)240 void JsThirdProviderInteractionOperation::SetSearchElementInfoByAccessibilityIdResult(
241     AccessibilityElementOperatorCallback& callback,
242     std::list<AccessibilityElementInfo>&& infos,
243     const int32_t requestId)
244 {
245     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
246     CHECK_NULL_VOID(jsAccessibilityManager);
247     auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
248     CHECK_NULL_VOID(context);
249     CHECK_NULL_VOID(context->GetTaskExecutor());
250     context->GetTaskExecutor()->PostTask(
251         [jsMgr = jsAccessibilityManager_,
252             infos = std::move(infos), &callback, requestId] () mutable {
253             auto jsAccessibilityManager = jsMgr.Upgrade();
254             CHECK_NULL_VOID(jsAccessibilityManager);
255             if (!jsAccessibilityManager->IsRegister()) {
256                 return;
257             }
258             jsAccessibilityManager->UpdateElementInfosTreeId(infos);
259             callback.SetSearchElementInfoByAccessibilityIdResult(infos, requestId);
260         }, TaskExecutor::TaskType::BACKGROUND, "SearchElementInfoByAccessibilityId");
261 }
262 
SearchElementInfoBySpecificProperty(const int64_t elementId,const SpecificPropertyParam & param,const int32_t requestId,AccessibilityElementOperatorCallback & callback)263 void JsThirdProviderInteractionOperation::SearchElementInfoBySpecificProperty(const int64_t elementId,
264     const SpecificPropertyParam &param, const int32_t requestId,
265     AccessibilityElementOperatorCallback &callback)
266 {
267     std::list<AccessibilityElementInfo> infos;
268     std::list<AccessibilityElementInfo> treeInfos;
269     callback.SetSearchElementInfoBySpecificPropertyResult(infos, treeInfos, requestId);
270 }
271 
SearchElementInfosByText(const int64_t elementId,const std::string & text,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)272 void JsThirdProviderInteractionOperation::SearchElementInfosByText(
273     const int64_t elementId, const std::string& text, const int32_t requestId,
274     Accessibility::AccessibilityElementOperatorCallback& callback)
275 {
276     // 1. Get real elementId
277     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
278     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
279     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
280         elementId, splitElementId, splitTreeId);
281 
282     // 2. FindAccessibilityNodeInfosByText from provider
283     std::list<Accessibility::AccessibilityElementInfo> infos;
284     bool ret = FindAccessibilityNodeInfosByTextFromProvider(
285         splitElementId, text, requestId, infos);
286     if (!ret) {
287         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
288             "SearchElementInfosByText failed.");
289         infos.clear();
290     }
291 
292     // 3. Return result
293     SetSearchElementInfoByTextResult(callback, std::move(infos), requestId);
294 }
295 
SearchDefaultFocusByWindowId(const int32_t windowId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback,const int32_t pageId)296 void JsThirdProviderInteractionOperation::SearchDefaultFocusByWindowId(
297     const int32_t windowId, const int32_t requestId,
298     Accessibility::AccessibilityElementOperatorCallback& callback, const int32_t pageId)
299 {
300 }
301 
FindAccessibilityNodeInfosByTextFromProvider(const int64_t splitElementId,const std::string & text,const int32_t requestId,std::list<Accessibility::AccessibilityElementInfo> & infos)302 bool JsThirdProviderInteractionOperation::FindAccessibilityNodeInfosByTextFromProvider(
303     const int64_t splitElementId, const std::string& text, const int32_t requestId,
304     std::list<Accessibility::AccessibilityElementInfo>& infos)
305 {
306     auto provider = accessibilityProvider_.Upgrade();
307     CHECK_NULL_RETURN(provider, false);
308     std::vector<ArkUI_AccessibilityElementInfo> nativeInfos;
309     int32_t code = provider->FindAccessibilityNodeInfosByText(
310         splitElementId, text, requestId, nativeInfos);
311     if (code != 0) {
312         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
313             "FindAccessibilityNodeInfosByTextFromProvider failed: %{public}d", code);
314         return false;
315     }
316 
317     NodeConfig config;
318     GetNodeConfig(config);
319     CopyNativeInfosToAccessibilityElementInfos(nativeInfos, config, infos);
320     return true;
321 }
322 
SetSearchElementInfoByTextResult(AccessibilityElementOperatorCallback & callback,std::list<AccessibilityElementInfo> && infos,const int32_t requestId)323 void JsThirdProviderInteractionOperation::SetSearchElementInfoByTextResult(
324     AccessibilityElementOperatorCallback& callback,
325     std::list<AccessibilityElementInfo>&& infos,
326     const int32_t requestId)
327 {
328     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
329     CHECK_NULL_VOID(jsAccessibilityManager);
330     auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
331     CHECK_NULL_VOID(context);
332     CHECK_NULL_VOID(context->GetTaskExecutor());
333     context->GetTaskExecutor()->PostTask(
334         [jsMgr = jsAccessibilityManager_,
335             infos = std::move(infos), &callback, requestId] () mutable {
336             auto jsAccessibilityManager = jsMgr.Upgrade();
337             CHECK_NULL_VOID(jsAccessibilityManager);
338             if (!jsAccessibilityManager->IsRegister()) {
339                 return;
340             }
341             jsAccessibilityManager->UpdateElementInfosTreeId(infos);
342             callback.SetSearchElementInfoByTextResult(infos, requestId);
343         }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
344 }
345 
FindFocusedElementInfo(const int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)346 void JsThirdProviderInteractionOperation::FindFocusedElementInfo(
347     const int64_t elementId, const int32_t focusType, const int32_t requestId,
348     Accessibility::AccessibilityElementOperatorCallback& callback)
349 {
350     // 1. Get real elementId
351     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
352     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
353     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
354         elementId, splitElementId, splitTreeId);
355 
356     // 2. FindFocusedAccessibilityNode from provider
357     Accessibility::AccessibilityElementInfo info;
358     bool ret = FindFocusedElementInfoFromProvider(
359         splitElementId, focusType, requestId, info);
360     if (!ret) {
361         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
362             "SearchElementInfosByText failed.");
363         info.SetValidElement(false);
364     }
365 
366     // 3. Return result
367     SetFindFocusedElementInfoResult(callback, info, requestId);
368 }
369 
FindFocusedElementInfoFromProvider(int64_t elementId,const int32_t focusType,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)370 bool JsThirdProviderInteractionOperation::FindFocusedElementInfoFromProvider(
371     int64_t elementId, const int32_t focusType, const int32_t requestId,
372     Accessibility::AccessibilityElementInfo& info)
373 {
374     auto provider = accessibilityProvider_.Upgrade();
375     CHECK_NULL_RETURN(provider, false);
376     ArkUI_AccessibilityElementInfo nativeInfo;
377     int32_t code = provider->FindFocusedAccessibilityNode(
378         elementId, focusType, requestId, nativeInfo);
379     if (code != 0) {
380         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
381             "FindFocusedElementInfoFromProvider failed: %{public}d", code);
382         return false;
383     }
384 
385     NodeConfig config;
386     GetNodeConfig(config);
387     CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
388     return true;
389 }
390 
SetFindFocusedElementInfoResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)391 void JsThirdProviderInteractionOperation::SetFindFocusedElementInfoResult(
392     AccessibilityElementOperatorCallback& callback,
393     AccessibilityElementInfo& info,
394     const int32_t requestId)
395 {
396     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
397     CHECK_NULL_VOID(jsAccessibilityManager);
398     jsAccessibilityManager->UpdateElementInfoTreeId(info);
399     callback.SetFindFocusedElementInfoResult(info, requestId);
400 }
401 
FocusMoveSearch(const int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)402 void JsThirdProviderInteractionOperation::FocusMoveSearch(
403     const int64_t elementId, const int32_t direction, const int32_t requestId,
404     Accessibility::AccessibilityElementOperatorCallback& callback)
405 {
406     // 1. Get real elementId
407     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
408     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
409     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
410         elementId, splitElementId, splitTreeId);
411 
412     // 2. FindNextFocusAccessibilityNode from provider
413     Accessibility::AccessibilityElementInfo info;
414     bool ret = FocusMoveSearchProvider(
415         splitElementId, direction, requestId, info);
416     // 3. Return result
417     if (!ret) {
418         auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
419         CHECK_NULL_VOID(jsAccessibilityManager);
420         auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
421         CHECK_NULL_VOID(context);
422         CHECK_NULL_VOID(context->GetTaskExecutor());
423         context->GetTaskExecutor()->PostTask(
424             [jsMgr = jsAccessibilityManager_, weakHost = host_, &callback, requestId] () {
425                 auto jsAccessibilityManager = jsMgr.Upgrade();
426                 if ((!jsAccessibilityManager) || (!jsAccessibilityManager->IsRegister())) {
427                     AccessibilityElementInfo nodeInfo;
428                     nodeInfo.SetValidElement(false);
429                     callback.SetFocusMoveSearchResult(nodeInfo, requestId);
430                     return;
431                 }
432                 jsAccessibilityManager->SetFocusMoveResultWithNode(weakHost, callback, requestId);
433             }, TaskExecutor::TaskType::UI, "AccessibilityThirdPartyFocusMoveSearchFail");
434     } else {
435         SetFocusMoveSearchResult(callback, info, requestId);
436     }
437 }
438 
FocusMoveSearchProvider(int64_t elementId,const int32_t direction,const int32_t requestId,Accessibility::AccessibilityElementInfo & info)439 bool JsThirdProviderInteractionOperation::FocusMoveSearchProvider(
440     int64_t elementId, const int32_t direction, const int32_t requestId,
441     Accessibility::AccessibilityElementInfo& info)
442 {
443     auto provider = accessibilityProvider_.Upgrade();
444     CHECK_NULL_RETURN(provider, false);
445     ArkUI_AccessibilityElementInfo nativeInfo;
446     int32_t code = provider->FindNextFocusAccessibilityNode(
447         elementId, direction, requestId, nativeInfo);
448     if (code != 0) {
449         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
450             "FocusMoveSearchProvider failed: %{public}d", code);
451         return false;
452     }
453 
454     NodeConfig config;
455     GetNodeConfig(config);
456     CopyNativeInfoToAccessibilityElementInfo(nativeInfo, config, info);
457     return true;
458 }
459 
SetFocusMoveSearchResult(AccessibilityElementOperatorCallback & callback,AccessibilityElementInfo & info,const int32_t requestId)460 void JsThirdProviderInteractionOperation::SetFocusMoveSearchResult(
461     AccessibilityElementOperatorCallback& callback,
462     AccessibilityElementInfo& info,
463     const int32_t requestId)
464 {
465     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
466     CHECK_NULL_VOID(jsAccessibilityManager);
467     jsAccessibilityManager->UpdateElementInfoTreeId(info);
468     callback.SetFocusMoveSearchResult(info, requestId);
469 }
470 
HandleActionWhenFindNodeFail(const int32_t action)471 void JsThirdProviderInteractionOperation::HandleActionWhenFindNodeFail(const int32_t action)
472 {
473     if (action == static_cast<int32_t>(
474         Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
475         ClearDrawBound();
476     }
477 }
478 
ExecuteAction(const int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)479 void JsThirdProviderInteractionOperation::ExecuteAction(
480     const int64_t elementId, const int32_t action,
481     const std::map<std::string, std::string>& actionArguments, const int32_t requestId,
482     Accessibility::AccessibilityElementOperatorCallback& callback)
483 {
484     // 1. Get real elementId
485     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
486     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
487     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
488         elementId, splitElementId, splitTreeId);
489 
490     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAction elementId: %{public}" PRId64 ","
491         " action: %{public}d, requestId: %{public}d, splitElementId: %{public}" PRId64 ","
492         " splitTreeId: %{public}d",
493         elementId, action, requestId, splitElementId, splitTreeId);
494     // 2. FindNextFocusAccessibilityNode from provider
495     std::list<Accessibility::AccessibilityElementInfo> infos;
496     bool ret = FindAccessibilityNodeInfosByIdFromProvider(
497         splitElementId, 0, requestId, infos, true); // for drawbound, no need fix host offset
498     if (!ret) {
499         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "Find info failed when ExecuteAction.");
500         SetExecuteActionResult(callback, false, requestId);
501         HandleActionWhenFindNodeFail(action);
502         return;
503     }
504 
505     // 3. DrawBound
506     ExecuteActionForThird(splitElementId, infos.front(), action);
507 
508     //4. ExecuteAccessibilityAction To provider
509     ret = ExecuteActionFromProvider(splitElementId, action, actionArguments, requestId);
510     if (!ret) {
511         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY, "ExecuteAccessibilityAction failed.");
512     }
513 
514     // 5. Return result
515     SetExecuteActionResult(callback, ret, requestId);
516 }
517 
ExecuteActionFromProvider(int64_t elementId,const int32_t action,const std::map<std::string,std::string> & actionArguments,const int32_t requestId)518 bool JsThirdProviderInteractionOperation::ExecuteActionFromProvider(
519     int64_t elementId, const int32_t action,
520     const std::map<std::string, std::string>& actionArguments, const int32_t requestId)
521 {
522     auto actionFilteredArguments = actionArguments;
523     RemoveKeysForClickAction(action, actionFilteredArguments);
524 
525     auto provider = accessibilityProvider_.Upgrade();
526     CHECK_NULL_RETURN(provider, false);
527     int32_t code = provider->ExecuteAccessibilityAction(
528         elementId, action, requestId, actionFilteredArguments);
529     if (code != 0) {
530         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
531             "ExecuteActionFromProvider failed: %{public}d", code);
532         return false;
533     }
534 
535     return true;
536 }
537 
ExecuteActionForThird(int64_t elementId,const AccessibilityElementInfo & nodeInfo,const int32_t action)538 bool JsThirdProviderInteractionOperation::ExecuteActionForThird(
539     int64_t elementId, const AccessibilityElementInfo& nodeInfo, const int32_t action)
540 {
541     auto hostNode = GetHost();
542     CHECK_NULL_RETURN(hostNode, false);
543     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
544     CHECK_NULL_RETURN(jsAccessibilityManager, false);
545     auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
546     CHECK_NULL_RETURN(context, false);
547     auto ngPipeline = AceType::DynamicCast<NG::PipelineContext>(context);
548     CHECK_NULL_RETURN(ngPipeline, false);
549     if (action == static_cast<int32_t>(
550         Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS)) {
551         jsAccessibilityManager->ActThirdAccessibilityFocus(
552             elementId, nodeInfo, hostNode, ngPipeline, false);
553     } else if (action == static_cast<int32_t>(
554         Accessibility::ActionType::ACCESSIBILITY_ACTION_CLEAR_ACCESSIBILITY_FOCUS)) {
555         jsAccessibilityManager->ActThirdAccessibilityFocus(
556             elementId, nodeInfo, hostNode, ngPipeline, true);
557     }
558     return true;
559 }
560 
SetExecuteActionResult(AccessibilityElementOperatorCallback & callback,const bool succeeded,const int32_t requestId)561 void JsThirdProviderInteractionOperation::SetExecuteActionResult(
562     AccessibilityElementOperatorCallback& callback,
563     const bool succeeded, const int32_t requestId)
564 {
565     callback.SetExecuteActionResult(succeeded, requestId);
566 }
567 
ClearFocus()568 void JsThirdProviderInteractionOperation::ClearFocus()
569 {
570     // 1. Clear focus from provider
571     ClearFocusFromProvider();
572     // 2. Clear DrawBound
573     ClearDrawBound();
574 }
575 
ClearFocusFromProvider()576 bool JsThirdProviderInteractionOperation::ClearFocusFromProvider()
577 {
578     auto provider = accessibilityProvider_.Upgrade();
579     CHECK_NULL_RETURN(provider, false);
580     int32_t code = provider->ClearFocusedAccessibilityNode();
581     if (code != 0) {
582         TAG_LOGW(AceLogTag::ACE_ACCESSIBILITY,
583             "ExecuteActionFromProvider failed: %{public}d", code);
584         return false;
585     }
586     return true;
587 }
588 
ClearDrawBound()589 bool JsThirdProviderInteractionOperation::ClearDrawBound()
590 {
591     auto jsAccessibilityManager = jsAccessibilityManager_.Upgrade();
592     CHECK_NULL_RETURN(jsAccessibilityManager, false);
593     auto hostNode = GetHost();
594     CHECK_NULL_RETURN(hostNode, false);
595     return jsAccessibilityManager->ClearThirdAccessibilityFocus(hostNode);
596 }
597 
OutsideTouch()598 void JsThirdProviderInteractionOperation::OutsideTouch()
599 {}
600 
GetCursorPosition(const int64_t elementId,const int32_t requestId,Accessibility::AccessibilityElementOperatorCallback & callback)601 void JsThirdProviderInteractionOperation::GetCursorPosition(
602     const int64_t elementId, const int32_t requestId,
603     Accessibility::AccessibilityElementOperatorCallback &callback)
604 {
605     auto provider = accessibilityProvider_.Upgrade();
606     CHECK_NULL_VOID(provider);
607     int32_t cursorPosition = -1;
608     int32_t code = provider->GetAccessibilityNodeCursorPosition(
609         elementId, requestId, cursorPosition);
610     if (code != 0) {
611         callback.SetCursorPositionResult(-1, requestId);
612         return;
613     }
614 
615     callback.SetCursorPositionResult(cursorPosition, requestId);
616 }
617 
GetHostRectTranslateInfo(NodeConfig & config)618 void JsThirdProviderInteractionOperation::GetHostRectTranslateInfo(NodeConfig& config)
619 {
620     auto host = host_.Upgrade();
621     if ((!host) || config.ignoreHostOffset) {
622         config.offset = NG::OffsetF(0, 0);
623         config.scaleX = 1.0f;
624         config.scaleY = 1.0f;
625         return;
626     }
627     auto rect = host->GetTransformRectRelativeToWindow();
628     NG::VectorF finalScale = host->GetTransformScaleRelativeToWindow();
629     auto pipeline = host->GetContextRefPtr();
630     if (pipeline) {
631         auto accessibilityManager = pipeline->GetAccessibilityManager();
632         if (accessibilityManager) {
633             auto windowInfo = accessibilityManager->GenerateWindowInfo(host, pipeline);
634             auto top = rect.Top() * windowInfo.scaleY + static_cast<int32_t>(windowInfo.top);
635             auto left = rect.Left() * windowInfo.scaleX + static_cast<int32_t>(windowInfo.left);
636             finalScale.x *= windowInfo.scaleX;
637             finalScale.y *= windowInfo.scaleY;
638             config.offset = NG::OffsetT(left, top);
639         } else {
640             auto windowRect = pipeline->GetDisplayWindowRectInfo();
641             auto top = rect.Top() + static_cast<int32_t>(windowRect.Top());
642             auto left = rect.Left() + static_cast<int32_t>(windowRect.Left());
643             config.offset = NG::OffsetT(left, top);
644         }
645     }
646 
647     config.scaleX = finalScale.x;
648     config.scaleY = finalScale.y;
649 }
650 
GetNodeConfig(NodeConfig & config)651 void JsThirdProviderInteractionOperation::GetNodeConfig(NodeConfig& config)
652 {
653     auto host = host_.Upgrade();
654     CHECK_NULL_VOID(host);
655     auto jsAccessibilityManager = GetHandler().Upgrade();
656     CHECK_NULL_VOID(jsAccessibilityManager);
657     auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
658     CHECK_NULL_VOID(context);
659     config.pageId = host->GetPageId();
660     config.windowId = static_cast<int32_t>(context->GetRealHostWindowId());
661     config.belongTreeId = belongTreeId_;
662     config.parentWindowId = static_cast<int32_t>(context->GetRealHostWindowId());
663     config.bundleName = AceApplicationInfo::GetInstance().GetPackageName();
664 
665     GetHostRectTranslateInfo(config);
666 }
667 
SetChildTreeIdAndWinId(const int64_t nodeId,const int32_t treeId,const int32_t childWindowId)668 void JsThirdProviderInteractionOperation::SetChildTreeIdAndWinId(
669     const int64_t nodeId, const int32_t treeId, const int32_t childWindowId)
670 {
671     TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "SetChildTreeIdAndWinId, node: %{public}" PRId64 ","
672         "treeId: %{public}d, childWindowId: %{public}d", nodeId, treeId, childWindowId);
673 }
674 
SetBelongTreeId(const int32_t treeId)675 void JsThirdProviderInteractionOperation::SetBelongTreeId(const int32_t treeId)
676 {
677     TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY, "SetBelongTreeId treeId: %{public}d", treeId);
678     belongTreeId_ = treeId;
679 }
680 
SendAccessibilityAsyncEventForThird(int64_t thirdElementId,Accessibility::EventType eventType)681 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEventForThird(
682     int64_t thirdElementId,
683     Accessibility::EventType eventType)
684 {
685     // 1. generate event
686     Accessibility::AccessibilityEventInfo event;
687     event.SetTimeStamp(GetMicroTickCount());
688     event.SetWindowChangeTypes(
689         Accessibility::WindowUpdateType::WINDOW_UPDATE_INVALID);
690     event.SetWindowContentChangeTypes(
691         Accessibility::WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_INVALID);
692     event.SetSource(thirdElementId);
693     event.SetEventType(eventType);
694     event.SetBundleName(AceApplicationInfo::GetInstance().GetPackageName());
695 
696     // 2. get element from third
697     // cut tree id
698     int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
699     int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
700     AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
701         thirdElementId, splitElementId, splitTreeId);
702     std::list<Accessibility::AccessibilityElementInfo> infos;
703     bool ret = FindAccessibilityNodeInfosByIdFromProvider(
704         splitElementId, 0, 0, infos);
705     if ((!ret) || (infos.size() == 0)) {
706         return -1;
707     }
708 
709     auto jsAccessibilityManager = GetHandler().Upgrade();
710     CHECK_NULL_RETURN(jsAccessibilityManager, -1);
711     jsAccessibilityManager->UpdateElementInfosTreeId(infos);
712     event.SetElementInfo(infos.front());
713 
714     // 3. change event info by host info
715     GetAccessibilityEventInfoFromNativeEvent(event, false);
716 
717     // 4. SendEvent
718     auto host = host_.Upgrade();
719     CHECK_NULL_RETURN(host, -1);
720     SendAccessibilitySyncEventToService(event, nullptr);
721     return 0;
722 }
723 
HandleNativeFocusUpdate(int64_t elementId,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)724 bool JsThirdProviderInteractionOperation::HandleNativeFocusUpdate(
725     int64_t elementId,
726     Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
727 {
728     ExecuteActionForThird(
729         elementId,
730         accessibilityEventInfo.GetElementInfo(),
731         static_cast<int32_t>(
732             Accessibility::ActionType::ACCESSIBILITY_ACTION_ACCESSIBILITY_FOCUS));
733     return false;
734 }
735 
HandleEventByFramework(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,Accessibility::AccessibilityEventInfo & accessibilityEventInfo)736 bool JsThirdProviderInteractionOperation::HandleEventByFramework(
737     const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
738     Accessibility::AccessibilityEventInfo& accessibilityEventInfo)
739 {
740     auto eventType = nativeAccessibilityEvent.GetEventType();
741     bool needSendEvent = true;
742     switch (eventType) {
743         case ArkUI_AccessibilityEventType::
744             ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_FOCUS_NODE_UPDATE:
745             if (nativeAccessibilityEvent.GetElementInfo()) {
746                 needSendEvent = HandleNativeFocusUpdate(
747                     nativeAccessibilityEvent.GetElementInfo()->GetElementId(),
748                     accessibilityEventInfo);
749             }
750             break;
751         default:
752             TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Unsupported eventType");
753     }
754     return needSendEvent;
755 }
756 
SendAccessibilityAsyncEvent(const ArkUI_AccessibilityEventInfo & nativeAccessibilityEvent,void (* callback)(int32_t errorCode))757 int32_t JsThirdProviderInteractionOperation::SendAccessibilityAsyncEvent(
758     const ArkUI_AccessibilityEventInfo& nativeAccessibilityEvent,
759     void (*callback)(int32_t errorCode))
760 {
761     bool ignoreHostOffset = CheckEventIgnoreHostOffset(nativeAccessibilityEvent);
762     // 1. Get OHOS::Accessibility::AccessibilityEventInfo
763     OHOS::Accessibility::AccessibilityEventInfo accessibilityEventInfo;
764     GetAccessibilityEventInfoFromNativeEvent(
765         nativeAccessibilityEvent, accessibilityEventInfo, ignoreHostOffset);
766 
767     // 2. handleEvent by frame work
768     bool needSendEvent =  HandleEventByFramework(
769         nativeAccessibilityEvent,
770         accessibilityEventInfo);
771     // 3. SendEvent
772     if (needSendEvent) {
773         auto jsAccessibilityManager = GetHandler().Upgrade();
774         CHECK_NULL_RETURN(jsAccessibilityManager, -1);
775         auto host = host_.Upgrade();
776         CHECK_NULL_RETURN(host, -1);
777         TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "Inner SendAccessibilityAsyncEvent");
778         SendAccessibilitySyncEventToService(accessibilityEventInfo, callback);
779     }
780     callback(0);
781     return 0;
782 }
783 
GetAccessibilityEventInfoFromNativeEvent(OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)784 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
785     OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
786     bool ignoreHostOffset)
787 {
788     // 1. Fill node config
789     NodeConfig config;
790     config.ignoreHostOffset = ignoreHostOffset;
791     GetNodeConfig(config);
792 
793     // 1.1. Fill elementInfo config
794     auto elementInfo = accessibilityEventInfo.GetElementInfo();
795     FillNodeConfig(config, elementInfo);
796     int64_t elementId = elementInfo.GetAccessibilityId();
797     AccessibilitySystemAbilityClient::SetSplicElementIdTreeId(belongTreeId_, elementId);
798     elementInfo.SetAccessibilityId(elementId);
799 
800     // 1.2. Fill eventInfo config
801     accessibilityEventInfo.SetPageId(config.pageId);
802     accessibilityEventInfo.SetWindowId(config.windowId);
803     accessibilityEventInfo.SetSource(elementId);
804     accessibilityEventInfo.SetComponentType(elementInfo.GetComponentType());
805     accessibilityEventInfo.AddContent(elementInfo.GetContent());
806     accessibilityEventInfo.SetElementInfo(elementInfo);
807     LogAccessibilityElementInfo("sendEvent", elementInfo);
808     LogAccessibilityEventInfo("sendEvent", accessibilityEventInfo);
809 }
810 
GetAccessibilityEventInfoFromNativeEvent(const ArkUI_AccessibilityEventInfo & nativeEventInfo,OHOS::Accessibility::AccessibilityEventInfo & accessibilityEventInfo,bool ignoreHostOffset)811 void JsThirdProviderInteractionOperation::GetAccessibilityEventInfoFromNativeEvent(
812     const ArkUI_AccessibilityEventInfo& nativeEventInfo,
813     OHOS::Accessibility::AccessibilityEventInfo& accessibilityEventInfo,
814     bool ignoreHostOffset)
815 {
816     // 1. Transform native event to OHOS::Accessibility::AccessibilityEventInfo
817     TransformAccessbilityEventInfo(
818         nativeEventInfo, accessibilityEventInfo);
819 
820     // 2. Transform Accessibility::AccessibilityEventInfo with host info
821     GetAccessibilityEventInfoFromNativeEvent(accessibilityEventInfo, ignoreHostOffset);
822 }
823 
SendAccessibilitySyncEventToService(const OHOS::Accessibility::AccessibilityEventInfo & eventInfo,void (* callback)(int32_t errorCode))824 bool JsThirdProviderInteractionOperation::SendAccessibilitySyncEventToService(
825     const OHOS::Accessibility::AccessibilityEventInfo& eventInfo,
826     [[maybe_unused]] void (*callback)(int32_t errorCode))
827 {
828     auto jsAccessibilityManager = GetHandler().Upgrade();
829     CHECK_NULL_RETURN(jsAccessibilityManager, false);
830     auto context = jsAccessibilityManager->GetPipelineContext().Upgrade();
831     CHECK_NULL_RETURN(context, false);
832     CHECK_NULL_RETURN(context->GetTaskExecutor(), false);
833     context->GetTaskExecutor()->PostTask(
834         [jsMgr = jsAccessibilityManager_, eventInfo] () mutable {
835             auto jsAccessibilityManager = jsMgr.Upgrade();
836             if (jsAccessibilityManager == nullptr) {
837                 return;
838             }
839 
840             if (!jsAccessibilityManager->IsRegister()) {
841                 return;
842             }
843 
844             auto client = AccessibilitySystemAbilityClient::GetInstance();
845             CHECK_NULL_VOID(client);
846             bool isEnabled = false;
847             client->IsEnabled(isEnabled);
848             if (!isEnabled) {
849                 return;
850             }
851 
852             TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY,
853                 "send accessibility componentType:%{public}s event:%{public}d accessibilityId:%{public}" PRId64,
854                 eventInfo.GetComponentType().c_str(), eventInfo.GetEventType(), eventInfo.GetAccessibilityId());
855             client->SendEvent(eventInfo);
856         }, TaskExecutor::TaskType::BACKGROUND, "SetSearchElementInfoByTextResult");
857     return true;
858 }
859 } // namespace OHOS::Ace::Framework
860