1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "js_third_provider_interaction_operation.h"
16
17 #include <algorithm>
18
19 #include "accessibility_constants.h"
20 #include "accessibility_event_info.h"
21 #include "accessibility_system_ability_client.h"
22 #include "adapter/ohos/entrance/ace_application_info.h"
23 #include "adapter/ohos/entrance/ace_container.h"
24 #include "base/log/ace_trace.h"
25 #include "base/log/dump_log.h"
26 #include "base/log/event_report.h"
27 #include "base/log/log.h"
28 #include "base/utils/linear_map.h"
29 #include "base/utils/string_utils.h"
30 #include "base/utils/utils.h"
31 #include "core/accessibility/accessibility_manager_ng.h"
32 #include "core/components_ng/base/inspector.h"
33 #include "core/components_v2/inspector/inspector_constants.h"
34 #include "core/pipeline/pipeline_context.h"
35 #include "core/pipeline_ng/pipeline_context.h"
36 #include "frameworks/bridge/common/dom/dom_type.h"
37 #include "frameworks/core/components_ng/pattern/web/web_pattern.h"
38 #include "js_accessibility_manager.h"
39 #include "js_third_accessibility_hover_ng.h"
40 #include "nlohmann/json.hpp"
41
42 using namespace OHOS::Accessibility;
43 using namespace OHOS::AccessibilityConfig;
44 using namespace std;
45
46 namespace OHOS::Ace::Framework {
47 constexpr int32_t ACCESSIBILITY_FOCUS_WITHOUT_EVENT = -2100001;
48
49 namespace {
isTouchExplorationEnabled(const RefPtr<NG::PipelineContext> & context)50 bool isTouchExplorationEnabled(const RefPtr<NG::PipelineContext>& context)
51 {
52 CHECK_NULL_RETURN(context, true);
53 auto jsAccessibilityManager = context->GetAccessibilityManager();
54 CHECK_NULL_RETURN(jsAccessibilityManager, true);
55 auto accessibilityWorkMode = jsAccessibilityManager->GenerateAccessibilityWorkMode();
56 return accessibilityWorkMode.isTouchExplorationEnabled;
57 }
58 } // namespace
59
GetElementInfoForThird(int64_t elementId,AccessibilityElementInfo & info,int64_t hostElementId)60 bool AccessibilityHoverManagerForThirdNG::GetElementInfoForThird(
61 int64_t elementId,
62 AccessibilityElementInfo& info,
63 int64_t hostElementId)
64 {
65 // this function only for third party hover process
66 auto jsThirdProviderOperator =
67 GetJsThirdProviderInteractionOperation(hostElementId).lock();
68 if (jsThirdProviderOperator == nullptr) {
69 TAG_LOGE(AceLogTag::ACE_ACCESSIBILITY,
70 "third jsThirdProviderOperator ptr is null, hostElementId %{public}" PRId64,
71 hostElementId);
72 return false;
73 }
74
75 std::list<Accessibility::AccessibilityElementInfo> infos;
76 bool ret = jsThirdProviderOperator->FindAccessibilityNodeInfosByIdFromProvider(
77 elementId, 0, 0, infos, true); // offset in hover no need fix host offset
78 if ((!ret) || (infos.size() == 0)) {
79 TAG_LOGE(AceLogTag::ACE_ACCESSIBILITY,
80 "cannot get third elementinfo :%{public}" PRId64 ", ret: %{public}d",
81 elementId, ret);
82 return false;
83 }
84 info = infos.front();
85 return true;
86 }
87
UpdateSearchStrategyByHitTestModeStr(std::string & hitTestMode,bool & shouldSearchSelf,bool & shouldSearchChildren)88 void AccessibilityHoverManagerForThirdNG::UpdateSearchStrategyByHitTestModeStr(
89 std::string& hitTestMode,
90 bool& shouldSearchSelf,
91 bool& shouldSearchChildren)
92 {
93 if (hitTestMode == "HitTestMode.Block") {
94 shouldSearchChildren = false;
95 } else if (hitTestMode == "HitTestMode.None") {
96 shouldSearchSelf = false;
97 }
98 }
99
HasAccessibilityTextOrDescription(const AccessibilityElementInfo & nodeInfo)100 bool AccessibilityHoverManagerForThirdNG::HasAccessibilityTextOrDescription(
101 const AccessibilityElementInfo& nodeInfo)
102 {
103 std::optional<std::string> accessibilityText = nodeInfo.GetAccessibilityText();
104 std::optional<std::string> accessibilityDescription = nodeInfo.GetDescriptionInfo();
105 return !accessibilityText.value_or("").empty() ||
106 !accessibilityDescription.value_or("").empty();
107 }
108
IsAccessibilityFocusable(const AccessibilityElementInfo & nodeInfo)109 bool AccessibilityHoverManagerForThirdNG::IsAccessibilityFocusable(
110 const AccessibilityElementInfo& nodeInfo)
111 {
112 auto level = nodeInfo.GetAccessibilityLevel();
113 if (level == NG::AccessibilityProperty::Level::YES_STR) {
114 return true;
115 }
116 if (level == NG::AccessibilityProperty::Level::NO_STR) {
117 return false;
118 }
119 if (nodeInfo.GetAccessibilityGroup() ||
120 !nodeInfo.GetActionList().empty() ||
121 HasAccessibilityTextOrDescription(nodeInfo) ||
122 !nodeInfo.GetContent().empty()) {
123 return true;
124 }
125 // expand to enabled and clickable
126 // default tag
127 if (NG::AccessibilityProperty::IsAccessibilityFocusableTag(
128 nodeInfo.GetComponentType()) == true) {
129 return true;
130 }
131 return false;
132 }
133
GetSearchStrategyForThird(const AccessibilityElementInfo & nodeInfo)134 std::pair<bool, bool> AccessibilityHoverManagerForThirdNG::GetSearchStrategyForThird(
135 const AccessibilityElementInfo& nodeInfo)
136 {
137 bool shouldSearchSelf = true;
138 bool shouldSearchChildren = true;
139 auto level = NG::AccessibilityProperty::Level::AUTO;
140 do {
141 level = nodeInfo.GetAccessibilityLevel();
142 bool hasAccessibilityText = HasAccessibilityTextOrDescription(nodeInfo);
143 if (level == NG::AccessibilityProperty::Level::YES_STR) {
144 break;
145 } else if (level == NG::AccessibilityProperty::Level::NO_HIDE_DESCENDANTS) {
146 shouldSearchSelf = false;
147 shouldSearchChildren = false;
148 break;
149 } else {
150 if (level == NG::AccessibilityProperty::Level::NO_STR) {
151 shouldSearchSelf = false;
152 } else {
153 // shouldSearchSelf is true here
154 if (hasAccessibilityText) {
155 break;
156 }
157 }
158 }
159
160 auto hitTestMode = nodeInfo.GetHitTestBehavior();
161 UpdateSearchStrategyByHitTestModeStr(
162 hitTestMode, shouldSearchSelf, shouldSearchChildren);
163 } while (0);
164
165 if (IsAccessibilityFocusable(nodeInfo) == false) {
166 shouldSearchSelf = false;
167 }
168
169 return std::make_pair(shouldSearchSelf, shouldSearchChildren);
170 }
171
172
HoverPathForThirdRecursive(const int64_t hostElementId,const NG::PointF & hoverPoint,const AccessibilityElementInfo & nodeInfo,AccessibilityHoverTestPathForThird & path)173 bool AccessibilityHoverManagerForThirdNG::HoverPathForThirdRecursive(
174 const int64_t hostElementId,
175 const NG::PointF& hoverPoint,
176 const AccessibilityElementInfo& nodeInfo,
177 AccessibilityHoverTestPathForThird& path)
178 {
179 bool hitTarget = false;
180 auto [shouldSearchSelf, shouldSearchChildren]
181 = GetSearchStrategyForThird(nodeInfo);
182 auto rectInScreen = nodeInfo.GetRectInScreen();
183 auto left = rectInScreen.GetLeftTopXScreenPostion();
184 auto right = rectInScreen.GetLeftTopYScreenPostion();
185 auto width = rectInScreen.GetRightBottomXScreenPostion() - rectInScreen.GetLeftTopXScreenPostion();
186 auto height = rectInScreen.GetRightBottomYScreenPostion() - rectInScreen.GetLeftTopYScreenPostion();
187 NG::RectF rect { left, right, width, height };
188 bool hitSelf = rect.IsInnerRegion(hoverPoint);
189 if (hitSelf && shouldSearchSelf) {
190 hitTarget = true;
191 path.push_back(nodeInfo.GetAccessibilityId());
192 }
193 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY,
194 "third hover elementId :%{public}" PRId64\
195 ", shouldSearchSelf: %{public}d shouldSearchChildren: %{public}d hitTarget: %{public}d ",
196 nodeInfo.GetAccessibilityId(), shouldSearchSelf, shouldSearchChildren, hitTarget);
197 if (shouldSearchChildren) {
198 auto childrenIds = nodeInfo.GetChildIds();
199 for (auto childId = childrenIds.rbegin(); childId != childrenIds.rend(); ++childId) {
200 AccessibilityElementInfo childInfo;
201 if (GetElementInfoForThird(*childId, childInfo, hostElementId) == false) {
202 break;
203 }
204 if (HoverPathForThirdRecursive(
205 hostElementId, hoverPoint, childInfo, path)) {
206 return true;
207 }
208 }
209 }
210 return hitTarget;
211 }
212
HoverPathForThird(const int64_t hostElementId,const NG::PointF & point,AccessibilityElementInfo & rootInfo)213 AccessibilityHoverTestPathForThird AccessibilityHoverManagerForThirdNG::HoverPathForThird(
214 const int64_t hostElementId,
215 const NG::PointF& point,
216 AccessibilityElementInfo& rootInfo)
217 {
218 AccessibilityHoverTestPathForThird path;
219 HoverPathForThirdRecursive(
220 hostElementId, point, rootInfo, path);
221 return path;
222 }
223
ResetHoverForThirdState()224 void AccessibilityHoverManagerForThirdNG::ResetHoverForThirdState()
225 {
226 hoverForThirdState_.idle = true;
227 hoverForThirdState_.nodesHovering.clear();
228 }
229
HandleAccessibilityHoverForThird(const AccessibilityHoverForThirdConfig & config)230 void AccessibilityHoverManagerForThirdNG::HandleAccessibilityHoverForThird(
231 const AccessibilityHoverForThirdConfig& config)
232 {
233 CHECK_NULL_VOID(config.hostNode);
234 if (config.eventType == NG::AccessibilityHoverEventType::ENTER) {
235 ResetHoverForThirdState();
236 }
237 std::vector<int64_t> currentNodesHovering;
238 std::vector<int64_t> lastNodesHovering = hoverForThirdState_.nodesHovering;
239 if (config.eventType != NG::AccessibilityHoverEventType::EXIT) {
240 AccessibilityElementInfo rootInfo;
241 if (GetElementInfoForThird(-1, rootInfo, config.hostElementId) == false) {
242 return;
243 }
244 AccessibilityHoverTestPathForThird path =
245 HoverPathForThird(config.hostElementId, config.point, rootInfo);
246 for (const auto& node: path) {
247 currentNodesHovering.push_back(node);
248 }
249 }
250 static constexpr int64_t INVALID_NODE_ID = -1;
251 int64_t lastHoveringId = INVALID_NODE_ID;
252 if (!lastNodesHovering.empty()) {
253 lastHoveringId = lastNodesHovering.back();
254 }
255 int64_t currentHoveringId = INVALID_NODE_ID;
256 if (!currentNodesHovering.empty()) {
257 currentHoveringId = currentNodesHovering.back();
258 }
259 auto jsThirdProviderOperator = GetJsThirdProviderInteractionOperation(
260 config.hostElementId).lock();
261 if (jsThirdProviderOperator == nullptr) {
262 TAG_LOGE(AceLogTag::ACE_ACCESSIBILITY, "jsThirdProviderOperator is null, "
263 "hostElementId %{public}" PRId64, config.hostElementId);
264 return;
265 }
266 if (lastHoveringId != INVALID_NODE_ID && lastHoveringId != currentHoveringId) {
267 jsThirdProviderOperator->SendAccessibilityAsyncEventForThird(lastHoveringId,
268 Accessibility::EventType::TYPE_VIEW_HOVER_EXIT_EVENT);
269 }
270 if ((currentHoveringId != INVALID_NODE_ID) && (currentHoveringId != lastHoveringId)) {
271 jsThirdProviderOperator->SendAccessibilityAsyncEventForThird(currentHoveringId,
272 Accessibility::EventType::TYPE_VIEW_HOVER_ENTER_EVENT);
273 }
274 hoverForThirdState_.nodesHovering = std::move(currentNodesHovering);
275 hoverForThirdState_.time = config.time;
276 hoverForThirdState_.source = config.sourceType;
277 hoverForThirdState_.idle = config.eventType == NG::AccessibilityHoverEventType::EXIT;
278 }
279
ClearThirdAccessibilityFocus(const RefPtr<NG::FrameNode> & hostNode)280 bool AccessibilityHoverManagerForThirdNG::ClearThirdAccessibilityFocus(
281 const RefPtr<NG::FrameNode>& hostNode)
282 {
283 CHECK_NULL_RETURN(hostNode, false);
284 RefPtr<NG::RenderContext> renderContext = hostNode->GetRenderContext();
285 CHECK_NULL_RETURN(renderContext, false);
286 renderContext->UpdateAccessibilityFocus(false);
287 return true;
288 }
289
ActThirdAccessibilityFocus(int64_t elementId,const AccessibilityElementInfo & nodeInfo,const RefPtr<NG::FrameNode> & hostNode,const RefPtr<NG::PipelineContext> & context,bool isNeedClear)290 bool AccessibilityHoverManagerForThirdNG::ActThirdAccessibilityFocus(
291 int64_t elementId,
292 const AccessibilityElementInfo& nodeInfo,
293 const RefPtr<NG::FrameNode>& hostNode,
294 const RefPtr<NG::PipelineContext>& context,
295 bool isNeedClear)
296 {
297 if (!isNeedClear && !isTouchExplorationEnabled(context)) {
298 TAG_LOGI(AceLogTag::ACE_ACCESSIBILITY, "third Accessibility focus or update focus but is not in touch mode");
299 return true;
300 }
301
302 CHECK_NULL_RETURN(hostNode, false);
303 RefPtr<NG::RenderContext> renderContext = nullptr;
304 renderContext = hostNode->GetRenderContext();
305 CHECK_NULL_RETURN(renderContext, false);
306 if (isNeedClear) {
307 renderContext->UpdateAccessibilityFocus(false);
308 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY,
309 "third act Accessibility element Id %{public}" PRId64 "Focus clear",
310 nodeInfo.GetAccessibilityId());
311 return true;
312 }
313 renderContext->UpdateAccessibilityFocus(false);
314 auto rectInScreen = nodeInfo.GetRectInScreen();
315 auto left = rectInScreen.GetLeftTopXScreenPostion();
316 auto right = rectInScreen.GetLeftTopYScreenPostion();
317 auto width = rectInScreen.GetRightBottomXScreenPostion() - rectInScreen.GetLeftTopXScreenPostion();
318 auto height = rectInScreen.GetRightBottomYScreenPostion() - rectInScreen.GetLeftTopYScreenPostion();
319 if ((width == 0) && (height == 0)) {
320 renderContext->UpdateAccessibilityFocus(false);
321 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY,
322 "third act Accessibility element Id %{public}" PRId64 "Focus clear by null rect",
323 nodeInfo.GetAccessibilityId());
324 return true;
325 }
326
327 NG::RectT<int32_t> rectInt { static_cast<int32_t>(left), static_cast<int32_t>(right),
328 static_cast<int32_t>(width), static_cast<int32_t>(height) };
329
330 renderContext->UpdateAccessibilityFocusRect(rectInt);
331 renderContext->UpdateAccessibilityFocus(true, ACCESSIBILITY_FOCUS_WITHOUT_EVENT);
332 TAG_LOGD(AceLogTag::ACE_ACCESSIBILITY,
333 "third act Accessibility element Id %{public}" PRId64 "Focus",
334 nodeInfo.GetAccessibilityId());
335 return true;
336 }
337
RegisterJsThirdProviderInteractionOperation(int64_t hostElementId,const std::shared_ptr<JsThirdProviderInteractionOperation> & jsThirdProviderOperator)338 void AccessibilityHoverManagerForThirdNG::RegisterJsThirdProviderInteractionOperation(
339 int64_t hostElementId,
340 const std::shared_ptr<JsThirdProviderInteractionOperation>& jsThirdProviderOperator)
341 {
342 jsThirdProviderOperator_[hostElementId] = jsThirdProviderOperator;
343 }
344
DeregisterJsThirdProviderInteractionOperation(int64_t hostElementId)345 void AccessibilityHoverManagerForThirdNG::DeregisterJsThirdProviderInteractionOperation(
346 int64_t hostElementId)
347 {
348 jsThirdProviderOperator_.erase(hostElementId);
349 }
350
351 namespace {
352 enum class DumpMode {
353 TREE,
354 NODE,
355 HANDLE_EVENT,
356 HOVER_TEST
357 };
358
359 struct DumpInfoArgument {
360 bool useWindowId = false;
361 DumpMode mode = DumpMode::TREE;
362 bool isDumpSimplify = false;
363 bool verbose = false;
364 int64_t rootId = -1;
365 int32_t pointX = 0;
366 int32_t pointY = 0;
367 int64_t nodeId = -1;
368 int32_t action = 0;
369 };
370
GetDumpInfoArgument(const std::vector<std::string> & params,DumpInfoArgument & argument)371 bool GetDumpInfoArgument(const std::vector<std::string>& params, DumpInfoArgument& argument)
372 {
373 argument.isDumpSimplify = params[0].compare("-simplify") == 0;
374 for (auto arg = params.begin() + 1; arg != params.end(); ++arg) {
375 if (*arg == "-w") {
376 argument.useWindowId = true;
377 } else if (*arg == "--root") {
378 ++arg;
379 if (arg == params.end()) {
380 DumpLog::GetInstance().Print(std::string("Error: --root is used to set the root node, ") +
381 "e.g. '--root ${AccessibilityId}'!");
382 return false;
383 }
384 argument.rootId = StringUtils::StringToLongInt(*arg);
385 } else if (*arg == "--hover-test") {
386 argument.mode = DumpMode::HOVER_TEST;
387 static constexpr int32_t NUM_POINT_DIMENSION = 2;
388 if (std::distance(arg, params.end()) <= NUM_POINT_DIMENSION) {
389 DumpLog::GetInstance().Print(std::string("Error: --hover-test is used to get nodes at a point ") +
390 "relative to the root node, e.g. '--hover-test ${x} ${y}'!");
391 return false;
392 }
393 ++arg;
394 argument.pointX = StringUtils::StringToInt(*arg);
395 ++arg;
396 argument.pointY = StringUtils::StringToInt(*arg);
397 } else if (*arg == "-v") {
398 argument.verbose = true;
399 } else if (*arg == "-json") {
400 argument.mode = DumpMode::TREE;
401 } else {
402 if (argument.mode == DumpMode::NODE) {
403 argument.mode = DumpMode::HANDLE_EVENT;
404 argument.action = StringUtils::StringToInt(*arg);
405 break;
406 } else {
407 argument.mode = DumpMode::NODE;
408 argument.nodeId = StringUtils::StringToLongInt(*arg);
409 }
410 }
411 }
412 return true;
413 }
414 } // namespace
415
DumpPropertyForThird(int64_t elementId,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager,const std::shared_ptr<JsThirdProviderInteractionOperation> & jsThirdProviderOperator)416 void AccessibilityHoverManagerForThirdNG::DumpPropertyForThird(
417 int64_t elementId,
418 const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager,
419 const std::shared_ptr<JsThirdProviderInteractionOperation>& jsThirdProviderOperator)
420 {
421 auto jsAccessibilityManagerTemp = jsAccessibilityManager.Upgrade();
422 CHECK_NULL_VOID(jsAccessibilityManagerTemp);
423 int64_t splitElementId = AccessibilityElementInfo::UNDEFINED_ACCESSIBILITY_ID;
424 int32_t splitTreeId = AccessibilityElementInfo::UNDEFINED_TREE_ID;
425 AccessibilitySystemAbilityClient::GetTreeIdAndElementIdBySplitElementId(
426 elementId, splitElementId, splitTreeId);
427 std::list<Accessibility::AccessibilityElementInfo> infos;
428 bool ret = jsThirdProviderOperator->FindAccessibilityNodeInfosByIdFromProvider(
429 splitElementId, 0, 0, infos);
430 if ((!ret) || (infos.size() == 0)) {
431 return;
432 }
433
434 Accessibility::AccessibilityElementInfo info = infos.front();
435 jsAccessibilityManagerTemp->DumpCommonPropertyNG(info, splitTreeId);
436 jsAccessibilityManagerTemp->DumpAccessibilityPropertyNG(info);
437 DumpLog::GetInstance().Print(0, info.GetComponentType(), info.GetChildCount());
438 }
439
OnDumpChildInfoForThirdRecursive(int64_t hostElementId,const std::vector<std::string> & params,std::vector<std::string> & info,const WeakPtr<JsAccessibilityManager> & jsAccessibilityManager)440 bool AccessibilityHoverManagerForThirdNG::OnDumpChildInfoForThirdRecursive(
441 int64_t hostElementId,
442 const std::vector<std::string>& params,
443 std::vector<std::string>& info,
444 const WeakPtr<JsAccessibilityManager>& jsAccessibilityManager)
445 {
446 DumpInfoArgument argument;
447 if (GetDumpInfoArgument(params, argument) == false) {
448 return true;
449 }
450 auto jsThirdProviderOperator =
451 GetJsThirdProviderInteractionOperation(hostElementId).lock();
452 if (jsThirdProviderOperator == nullptr) {
453 return true;
454 }
455 switch (argument.mode) {
456 case DumpMode::NODE:
457 DumpPropertyForThird(argument.nodeId, jsAccessibilityManager, jsThirdProviderOperator);
458 break;
459 case DumpMode::TREE:
460 case DumpMode::HANDLE_EVENT:
461 case DumpMode::HOVER_TEST:
462 default:
463 DumpLog::GetInstance().Print("Error: invalid arguments!");
464 break;
465 }
466 return true;
467 }
468
469
470 } // namespace OHOS::Ace::Framework
471