1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #include <atomic> 17 #include <chrono> 18 #include <fstream> 19 #include <memory> 20 #include <iostream> 21 #include <thread> 22 #include <utility> 23 #include <condition_variable> 24 #include "accessibility_event_info.h" 25 #include "accessibility_ui_test_ability.h" 26 #include "ability_manager_client.h" 27 #include "display_manager.h" 28 #include "screen_manager.h" 29 #include "input_manager.h" 30 #include "png.h" 31 #include "wm_common.h" 32 #include "system_ui_controller.h" 33 34 using namespace std; 35 using namespace chrono; 36 37 namespace OHOS::uitest { 38 using namespace std; 39 using namespace nlohmann; 40 using namespace OHOS::MMI; 41 using namespace OHOS::Accessibility; 42 using namespace OHOS::Rosen; 43 using namespace OHOS::Media; 44 45 constexpr auto g_sceneboardId = 1; 46 47 class UiEventMonitor final : public AccessibleAbilityListener { 48 public: 49 virtual ~UiEventMonitor() override = default; 50 51 void OnAbilityConnected() override; 52 53 void OnAbilityDisconnected() override; 54 55 void OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) override; 56 57 void SetOnAbilityConnectCallback(function<void()> onConnectCb); 58 59 void SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb); 60 OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> & keyEvent)61 bool OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> &keyEvent) override 62 { 63 return false; 64 } 65 66 uint64_t GetLastEventMillis(); 67 68 bool WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs); 69 70 void RegisterUiEventListener(shared_ptr<UiEventListener> listerner); 71 72 private: 73 function<void()> onConnectCallback_ = nullptr; 74 function<void()> onDisConnectCallback_ = nullptr; 75 atomic<uint64_t> lastEventMillis_ = 0; 76 vector<shared_ptr<UiEventListener>> listeners_; 77 }; 78 79 struct EventSpec { 80 std::string_view componentTyep; 81 int32_t eventType; 82 std::string_view event; 83 }; 84 85 static constexpr EventSpec WATCHED_EVENTS[] = { 86 {"Toast", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "toastShow"}, 87 {"AlertDialog", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "dialogShow"} 88 }; 89 GetWatchedEvent(const AccessibilityEventInfo & eventInfo)90 static std::string GetWatchedEvent(const AccessibilityEventInfo &eventInfo) 91 { 92 for (unsigned long index = 0; index < sizeof(WATCHED_EVENTS) / sizeof(EventSpec); index++) { 93 if (WATCHED_EVENTS[index].componentTyep == eventInfo.GetComponentType() && 94 WATCHED_EVENTS[index].eventType == eventInfo.GetWindowContentChangeTypes()) { 95 LOG_W("Capture event: %{public}s", WATCHED_EVENTS[index].event.data()); 96 return string(WATCHED_EVENTS[index].event); 97 } 98 } 99 return "undefine"; 100 } 101 102 // UiEventMonitor instance. 103 static shared_ptr<UiEventMonitor> g_monitorInstance_; 104 SetOnAbilityConnectCallback(function<void ()> onConnectCb)105 void UiEventMonitor::SetOnAbilityConnectCallback(function<void()> onConnectCb) 106 { 107 onConnectCallback_ = std::move(onConnectCb); 108 } 109 SetOnAbilityDisConnectCallback(function<void ()> onDisConnectCb)110 void UiEventMonitor::SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb) 111 { 112 onDisConnectCallback_ = std::move(onDisConnectCb); 113 } 114 OnAbilityConnected()115 void UiEventMonitor::OnAbilityConnected() 116 { 117 if (onConnectCallback_ != nullptr) { 118 onConnectCallback_(); 119 } 120 } 121 OnAbilityDisconnected()122 void UiEventMonitor::OnAbilityDisconnected() 123 { 124 if (onDisConnectCallback_ != nullptr) { 125 onDisConnectCallback_(); 126 } 127 } 128 129 // the monitored events 130 static constexpr uint32_t EVENT_MASK = EventType::TYPE_VIEW_TEXT_UPDATE_EVENT | 131 EventType::TYPE_PAGE_STATE_UPDATE | 132 EventType::TYPE_PAGE_CONTENT_UPDATE | 133 EventType::TYPE_VIEW_SCROLLED_EVENT | 134 EventType::TYPE_WINDOW_UPDATE; 135 RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner)136 void UiEventMonitor::RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner) 137 { 138 listeners_.emplace_back(listerner); 139 } 140 OnAccessibilityEvent(const AccessibilityEventInfo & eventInfo)141 void UiEventMonitor::OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) 142 { 143 LOG_W("OnEvent:0x%{public}x", eventInfo.GetEventType()); 144 auto capturedEvent = GetWatchedEvent(eventInfo); 145 if (capturedEvent != "undefine") { 146 auto bundleName = eventInfo.GetBundleName(); 147 auto contentList = eventInfo.GetContentList(); 148 auto text = !contentList.empty() ? contentList[0] : ""; 149 auto type = eventInfo.GetComponentType(); 150 UiEventSourceInfo uiEventSourceInfo = {bundleName, text, type}; 151 for (auto &listener : listeners_) { 152 listener->OnEvent(capturedEvent, uiEventSourceInfo); 153 } 154 } 155 if ((eventInfo.GetEventType() & EVENT_MASK) > 0) { 156 lastEventMillis_.store(GetCurrentMillisecond()); 157 } 158 } 159 GetLastEventMillis()160 uint64_t UiEventMonitor::GetLastEventMillis() 161 { 162 if (lastEventMillis_.load() <= 0) { 163 lastEventMillis_.store(GetCurrentMillisecond()); 164 } 165 return lastEventMillis_.load(); 166 } 167 WaitEventIdle(uint32_t idleThresholdMs,uint32_t timeoutMs)168 bool UiEventMonitor::WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs) 169 { 170 const auto currentMs = GetCurrentMillisecond(); 171 if (lastEventMillis_.load() <= 0) { 172 lastEventMillis_.store(currentMs); 173 } 174 if (currentMs - lastEventMillis_.load() >= idleThresholdMs) { 175 return true; 176 } 177 static constexpr auto sliceMs = 10; 178 this_thread::sleep_for(chrono::milliseconds(sliceMs)); 179 if (timeoutMs <= sliceMs) { 180 return false; 181 } 182 return WaitEventIdle(idleThresholdMs, timeoutMs - sliceMs); 183 } 184 SysUiController()185 SysUiController::SysUiController() : UiController() {} 186 ~SysUiController()187 SysUiController::~SysUiController() 188 { 189 DisConnectFromSysAbility(); 190 } 191 Initialize()192 bool SysUiController::Initialize() 193 { 194 return this->ConnectToSysAbility(); 195 } 196 GenerateNodeHash(AccessibilityElementInfo & node)197 static size_t GenerateNodeHash(AccessibilityElementInfo &node) 198 { 199 static constexpr auto SHIFT_BITS = 32U; 200 static constexpr auto hashFunc = hash<string>(); 201 int64_t intId = node.GetWindowId(); 202 intId = (intId << SHIFT_BITS) + node.GetAccessibilityId(); 203 const string strId = node.GetBundleName() + node.GetComponentType() + to_string(intId); 204 return hashFunc(strId); 205 } 206 GetVisibleRect(Rect windowBounds,Accessibility::Rect nodeBounds)207 static Rect GetVisibleRect(Rect windowBounds, Accessibility::Rect nodeBounds) 208 { 209 auto leftX = nodeBounds.GetLeftTopXScreenPostion(); 210 auto topY = nodeBounds.GetLeftTopYScreenPostion(); 211 auto rightX = nodeBounds.GetRightBottomXScreenPostion(); 212 auto bottomY = nodeBounds.GetRightBottomYScreenPostion(); 213 Rect newBounds((leftX < windowBounds.left_) ? windowBounds.left_ : leftX, 214 (rightX > windowBounds.right_) ? windowBounds.right_ : rightX, 215 (topY < windowBounds.top_) ? windowBounds.top_ : topY, 216 (bottomY > windowBounds.bottom_) ? windowBounds.bottom_ : bottomY); 217 return newBounds; 218 } 219 MarshalAccessibilityNodeAttributes(AccessibilityElementInfo & node,json & to,Rect windowBounds)220 static void MarshalAccessibilityNodeAttributes(AccessibilityElementInfo &node, json &to, Rect windowBounds) 221 { 222 to[ATTR_NAMES[UiAttr::HASHCODE].data()] = to_string(GenerateNodeHash(node)); 223 to[ATTR_NAMES[UiAttr::TEXT].data()] = node.GetContent(); 224 to[ATTR_NAMES[UiAttr::ACCESSIBILITY_ID].data()] = to_string(node.GetAccessibilityId()); 225 to[ATTR_NAMES[UiAttr::ID].data()] = node.GetInspectorKey(); 226 to[ATTR_NAMES[UiAttr::KEY].data()] = node.GetInspectorKey(); 227 to[ATTR_NAMES[UiAttr::TYPE].data()] = node.GetComponentType(); 228 to[ATTR_NAMES[UiAttr::ENABLED].data()] = node.IsEnabled() ? "true" : "false"; 229 to[ATTR_NAMES[UiAttr::FOCUSED].data()] = node.IsFocused() ? "true" : "false"; 230 to[ATTR_NAMES[UiAttr::SELECTED].data()] = node.IsSelected() ? "true" : "false"; 231 to[ATTR_NAMES[UiAttr::CHECKABLE].data()] = node.IsCheckable() ? "true" : "false"; 232 to[ATTR_NAMES[UiAttr::CHECKED].data()] = node.IsChecked() ? "true" : "false"; 233 to[ATTR_NAMES[UiAttr::CLICKABLE].data()] = "false"; 234 to[ATTR_NAMES[UiAttr::LONG_CLICKABLE].data()] = "false"; 235 to[ATTR_NAMES[UiAttr::SCROLLABLE].data()] = "false"; 236 to[ATTR_NAMES[UiAttr::HOST_WINDOW_ID].data()] = to_string(node.GetWindowId()); 237 to[ATTR_NAMES[UiAttr::VISIBLE].data()] = "false"; 238 const auto bounds = node.GetRectInScreen(); 239 const auto rect = GetVisibleRect(windowBounds, bounds); 240 stringstream stream; 241 // "[%d,%d][%d,%d]", rect.left, rect.top, rect.right, rect.bottom 242 stream << "[" << rect.left_ << "," << rect.top_ << "]" << "[" << rect.right_ << "," << rect.bottom_ << "]"; 243 to[ATTR_NAMES[UiAttr::BOUNDS].data()] = stream.str(); 244 to[ATTR_NAMES[UiAttr::ORIGBOUNDS].data()] = stream.str(); 245 auto actionList = node.GetActionList(); 246 for (auto &action : actionList) { 247 switch (action.GetActionType()) { 248 case ACCESSIBILITY_ACTION_CLICK: 249 to[ATTR_NAMES[UiAttr::CLICKABLE].data()] = "true"; 250 break; 251 case ACCESSIBILITY_ACTION_LONG_CLICK: 252 to[ATTR_NAMES[UiAttr::LONG_CLICKABLE].data()] = "true"; 253 break; 254 case ACCESSIBILITY_ACTION_SCROLL_FORWARD: 255 case ACCESSIBILITY_ACTION_SCROLL_BACKWARD: 256 to[ATTR_NAMES[UiAttr::SCROLLABLE].data()] = "true"; 257 break; 258 default: 259 break; 260 } 261 } 262 } 263 MarshallAccessibilityNodeInfo(AccessibilityElementInfo & from,json & to,int32_t index,Rect windowBounds,bool visitChild)264 static void MarshallAccessibilityNodeInfo(AccessibilityElementInfo &from, json &to, int32_t index, 265 Rect windowBounds, bool visitChild) 266 { 267 json attributes; 268 MarshalAccessibilityNodeAttributes(from, attributes, windowBounds); 269 if (from.GetComponentType() == "rootdecortag" || from.GetInspectorKey() == "ContainerModalTitleRow") { 270 attributes[ATTR_NAMES[UiAttr::TYPE].data()] = "DecorBar"; 271 } 272 attributes["index"] = to_string(index); 273 to["attributes"] = attributes; 274 auto childList = json::array(); 275 if (!visitChild) { 276 to["children"] = childList; 277 return; 278 } 279 const auto childCount = from.GetChildCount(); 280 AccessibilityElementInfo child; 281 auto ability = AccessibilityUITestAbility::GetInstance(); 282 for (auto idx = 0; idx < childCount; idx++) { 283 auto ret = ability->GetChildElementInfo(idx, from, child); 284 if (ret == RET_OK) { 285 if (!child.IsVisible()) { 286 LOG_I("This node is not visible, node Id: %{public}d", child.GetAccessibilityId()); 287 continue; 288 } 289 auto parcel = json(); 290 MarshallAccessibilityNodeInfo(child, parcel, idx, windowBounds, visitChild); 291 childList.push_back(parcel); 292 } else { 293 LOG_W("Get Node child at index=%{public}d failed", idx); 294 } 295 } 296 to["children"] = childList; 297 } 298 InflateWindowInfo(AccessibilityWindowInfo & node,Window & info)299 static void InflateWindowInfo(AccessibilityWindowInfo& node, Window& info) 300 { 301 info.focused_ = node.IsFocused(); 302 info.actived_ = node.IsActive(); 303 info.decoratorEnabled_ = node.IsDecorEnable(); 304 info.mode_ = WindowMode::UNKNOWN; 305 const auto origMode = static_cast<OHOS::Rosen::WindowMode>(node.GetWindowMode()); 306 switch (origMode) { 307 case OHOS::Rosen::WindowMode::WINDOW_MODE_FULLSCREEN: 308 info.mode_ = WindowMode::FULLSCREEN; 309 break; 310 case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_PRIMARY: 311 info.mode_ = WindowMode::SPLIT_PRIMARY; 312 break; 313 case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_SECONDARY: 314 info.mode_ = WindowMode::SPLIT_SECONDARY; 315 break; 316 case OHOS::Rosen::WindowMode::WINDOW_MODE_FLOATING: 317 info.mode_ = WindowMode::FLOATING; 318 break; 319 case OHOS::Rosen::WindowMode::WINDOW_MODE_PIP: 320 info.mode_ = WindowMode::PIP; 321 break; 322 default: 323 info.mode_ = WindowMode::UNKNOWN; 324 break; 325 } 326 } 327 GetAamsWindowInfos(vector<AccessibilityWindowInfo> & windows)328 static bool GetAamsWindowInfos(vector<AccessibilityWindowInfo> &windows) 329 { 330 auto ability = AccessibilityUITestAbility::GetInstance(); 331 if (ability->GetWindows(windows) != RET_OK) { 332 LOG_W("GetWindows from AccessibilityUITestAbility failed"); 333 return false; 334 } 335 sort(windows.begin(), windows.end(), [](auto &w1, auto &w2) -> bool { 336 if (w1.GetWindowId() == g_sceneboardId) { 337 return false; 338 } else if (w2.GetWindowId() == g_sceneboardId) { 339 return true; 340 } 341 return w1.GetWindowLayer() > w2.GetWindowLayer(); 342 }); 343 return true; 344 } 345 GetUiHierarchy(vector<pair<Window,nlohmann::json>> & out,bool getWidgetNodes,string targetApp)346 void SysUiController::GetUiHierarchy(vector<pair<Window, nlohmann::json>> &out, bool getWidgetNodes, 347 string targetApp) 348 { 349 static mutex dumpMutex; // disallow concurrent dumpUi 350 if (!connected_ && !ConnectToSysAbility()) { 351 LOG_W("Connect to AccessibilityUITestAbility failed"); 352 return; 353 } 354 dumpMutex.lock(); 355 vector<AccessibilityWindowInfo> windows; 356 if (!GetAamsWindowInfos(windows)) { 357 dumpMutex.unlock(); 358 return; 359 } 360 auto screenSize = GetDisplaySize(); 361 AccessibilityElementInfo elementInfo; 362 const auto foreAbility = AAFwk::AbilityManagerClient::GetInstance()->GetTopAbility(); 363 vector<Rect> overlays; 364 for (auto &window : windows) { 365 if (AccessibilityUITestAbility::GetInstance()->GetRootByWindow(window, elementInfo) == RET_OK) { 366 const auto app = elementInfo.GetBundleName(); 367 LOG_D("Get window at layer %{public}d, appId: %{public}s, windowId: %{public}d", 368 window.GetWindowLayer(), app.c_str(), window.GetWindowId()); 369 if (targetApp != "" && app != targetApp) { 370 continue; 371 } 372 // apply window bounds as root node bounds 373 auto screenRect = Rect(0, screenSize.px_, 0, screenSize.py_); 374 auto boundsInScreen = GetVisibleRect(screenRect, window.GetRectInScreen()); 375 auto winInfo = Window(window.GetWindowId()); 376 InflateWindowInfo(window, winInfo); 377 winInfo.bounds_ = (window.GetWindowId() == g_sceneboardId) ? screenRect : boundsInScreen; 378 winInfo.bundleName_ = app; 379 Rect visibleArea = winInfo.bounds_; 380 if (!RectAlgorithm::ComputeMaxVisibleRegion(winInfo.bounds_, overlays, visibleArea)) { 381 continue; 382 } 383 auto root = nlohmann::json(); 384 root["bundleName"] = app; 385 root["abilityName"] = (app == foreAbility.GetBundleName()) ? foreAbility.GetAbilityName() : ""; 386 root["pagePath"] = (app == foreAbility.GetBundleName()) ? elementInfo.GetPagePath() : ""; 387 MarshallAccessibilityNodeInfo(elementInfo, root, 0, winInfo.bounds_, getWidgetNodes); 388 overlays.push_back(winInfo.bounds_); 389 out.push_back(make_pair(move(winInfo), move(root))); 390 LOG_D("Get node at layer %{public}d, appId: %{public}s", window.GetWindowLayer(), app.c_str()); 391 } else { 392 LOG_W("GetRootByWindow failed, windowId: %{public}d", window.GetWindowId()); 393 } 394 } 395 dumpMutex.unlock(); 396 } 397 InjectTouchEventSequence(const PointerMatrix & events) const398 void SysUiController::InjectTouchEventSequence(const PointerMatrix &events) const 399 { 400 for (uint32_t step = 0; step < events.GetSteps(); step++) { 401 auto pointerEvent = PointerEvent::Create(); 402 for (uint32_t finger = 0; finger < events.GetFingers(); finger++) { 403 pointerEvent->SetPointerId(finger); 404 PointerEvent::PointerItem pinterItem; 405 pinterItem.SetPointerId(finger); 406 pinterItem.SetDisplayX(events.At(finger, step).point_.px_); 407 pinterItem.SetDisplayY(events.At(finger, step).point_.py_); 408 switch (events.At(finger, step).stage_) { 409 case ActionStage::DOWN: 410 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_DOWN); 411 break; 412 case ActionStage::MOVE: 413 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_MOVE); 414 break; 415 case ActionStage::UP: 416 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_UP); 417 break; 418 } 419 pinterItem.SetPressed(events.At(finger, step).stage_ != ActionStage::UP); 420 pointerEvent->AddPointerItem(pinterItem); 421 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_TOUCHSCREEN); 422 DisplayManager &displayMgr = DisplayManager::GetInstance(); 423 pointerEvent->SetTargetDisplayId(displayMgr.GetDefaultDisplayId()); 424 InputManager::GetInstance()->SimulateInputEvent(pointerEvent); 425 if (events.At(finger, step).holdMs_ > 0) { 426 this_thread::sleep_for(chrono::milliseconds(events.At(finger, step).holdMs_)); 427 } 428 } 429 } 430 } 431 CreateMouseActionEvent(MouseOpArgs mouseOpArgs,MouseEventType action)432 static std::shared_ptr<OHOS::MMI::PointerEvent> CreateMouseActionEvent(MouseOpArgs mouseOpArgs, 433 MouseEventType action) 434 { 435 auto pointerEvent = PointerEvent::Create(); 436 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_MOUSE); 437 pointerEvent->SetPointerId(0); 438 switch (action) { 439 case MouseEventType::M_MOVE: 440 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_MOVE); 441 break; 442 case MouseEventType::AXIS_BEGIN: 443 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_BEGIN); 444 break; 445 case MouseEventType::AXIS_END: 446 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_END); 447 break; 448 case MouseEventType::BUTTON_DOWN: 449 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_DOWN); 450 break; 451 case MouseEventType::BUTTON_UP: 452 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_UP); 453 break; 454 default: 455 break; 456 } 457 PointerEvent::PointerItem item; 458 if (action == MouseEventType::AXIS_BEGIN || action == MouseEventType::AXIS_END) { 459 constexpr double axialValue = 15; 460 auto injectAxialValue = mouseOpArgs.adown_ ? axialValue : -axialValue; 461 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, injectAxialValue); 462 item.SetDownTime(0); 463 } else if (action == MouseEventType::BUTTON_DOWN || action == MouseEventType::BUTTON_UP) { 464 pointerEvent->SetButtonId(mouseOpArgs.button_); 465 pointerEvent->SetButtonPressed(mouseOpArgs.button_); 466 } 467 item.SetDisplayX(mouseOpArgs.point_.px_); 468 item.SetDisplayY(mouseOpArgs.point_.py_); 469 item.SetPressed(action == MouseEventType::BUTTON_DOWN); 470 pointerEvent->AddPointerItem(item); 471 return pointerEvent; 472 } 473 CreateSingleKeyEvent(int32_t key,ActionStage action)474 static std::shared_ptr<OHOS::MMI::KeyEvent> CreateSingleKeyEvent(int32_t key, ActionStage action) 475 { 476 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 477 keyEvent->SetKeyCode(key); 478 switch (action) { 479 case ActionStage::DOWN: 480 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_DOWN); 481 break; 482 case ActionStage::UP: 483 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_UP); 484 break; 485 default: 486 break; 487 } 488 OHOS::MMI::KeyEvent::KeyItem keyItem; 489 keyItem.SetKeyCode(key); 490 keyItem.SetPressed(action == ActionStage::DOWN); 491 keyEvent->AddKeyItem(keyItem); 492 return keyEvent; 493 } 494 InjectMouseClick(MouseOpArgs mouseOpArgs) const495 void SysUiController::InjectMouseClick(MouseOpArgs mouseOpArgs) const 496 { 497 constexpr uint32_t focusTimeMs = 40; 498 auto mouseMove = CreateMouseActionEvent(mouseOpArgs, MouseEventType::M_MOVE); 499 InputManager::GetInstance()->SimulateInputEvent(mouseMove); 500 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 501 if (mouseOpArgs.key1_ != KEYCODE_NONE) { 502 auto dwonEvent1 = CreateSingleKeyEvent(mouseOpArgs.key1_, ActionStage::DOWN); 503 InputManager::GetInstance()->SimulateInputEvent(dwonEvent1); 504 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 505 if (mouseOpArgs.key2_ != KEYCODE_NONE) { 506 auto dwonEvent2 = CreateSingleKeyEvent(mouseOpArgs.key2_, ActionStage::DOWN); 507 InputManager::GetInstance()->SimulateInputEvent(dwonEvent2); 508 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 509 } 510 } 511 auto mouseDown = CreateMouseActionEvent(mouseOpArgs, MouseEventType::BUTTON_DOWN); 512 InputManager::GetInstance()->SimulateInputEvent(mouseDown); 513 auto mouseUp = CreateMouseActionEvent(mouseOpArgs, MouseEventType::BUTTON_UP); 514 InputManager::GetInstance()->SimulateInputEvent(mouseUp); 515 if (mouseOpArgs.key2_ != KEYCODE_NONE) { 516 auto upEvent = CreateSingleKeyEvent(mouseOpArgs.key2_, ActionStage::UP); 517 InputManager::GetInstance()->SimulateInputEvent(upEvent); 518 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 519 } 520 if (mouseOpArgs.key1_ != KEYCODE_NONE) { 521 auto upEvent = CreateSingleKeyEvent(mouseOpArgs.key1_, ActionStage::UP); 522 InputManager::GetInstance()->SimulateInputEvent(upEvent); 523 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 524 } 525 } 526 InjectMouseScroll(MouseOpArgs mouseOpArgs) const527 void SysUiController::InjectMouseScroll(MouseOpArgs mouseOpArgs) const 528 { 529 if (mouseOpArgs.key1_ != KEYCODE_NONE) { 530 auto dwonEvent1 = CreateSingleKeyEvent(mouseOpArgs.key1_, ActionStage::DOWN); 531 InputManager::GetInstance()->SimulateInputEvent(dwonEvent1); 532 if (mouseOpArgs.key2_ != KEYCODE_NONE) { 533 auto dwonEvent2 = CreateSingleKeyEvent(mouseOpArgs.key2_, ActionStage::DOWN); 534 InputManager::GetInstance()->SimulateInputEvent(dwonEvent2); 535 } 536 } 537 auto mouseMove = CreateMouseActionEvent(mouseOpArgs, MouseEventType::M_MOVE); 538 InputManager::GetInstance()->SimulateInputEvent(mouseMove); 539 constexpr uint32_t focusTimeMs = 40; 540 for (auto index = 0; index < mouseOpArgs.scrollValue_; index++) { 541 auto mouseScroll1 = CreateMouseActionEvent(mouseOpArgs, MouseEventType::AXIS_BEGIN); 542 InputManager::GetInstance()->SimulateInputEvent(mouseScroll1); 543 auto mouseScroll2 = CreateMouseActionEvent(mouseOpArgs, MouseEventType::AXIS_END); 544 InputManager::GetInstance()->SimulateInputEvent(mouseScroll2); 545 this_thread::sleep_for(chrono::milliseconds(focusTimeMs)); 546 } 547 if (mouseOpArgs.key2_ != KEYCODE_NONE) { 548 auto upEvent = CreateSingleKeyEvent(mouseOpArgs.key2_, ActionStage::UP); 549 InputManager::GetInstance()->SimulateInputEvent(upEvent); 550 } 551 if (mouseOpArgs.key1_ != KEYCODE_NONE) { 552 auto upEvent = CreateSingleKeyEvent(mouseOpArgs.key1_, ActionStage::UP); 553 InputManager::GetInstance()->SimulateInputEvent(upEvent); 554 } 555 } 556 InjectMouseMove(MouseOpArgs mouseOpArgs) const557 void SysUiController::InjectMouseMove(MouseOpArgs mouseOpArgs) const 558 { 559 auto event = CreateMouseActionEvent(mouseOpArgs, MouseEventType::M_MOVE); 560 InputManager::GetInstance()->SimulateInputEvent(event); 561 } 562 InjectKeyEventSequence(const vector<KeyEvent> & events) const563 void SysUiController::InjectKeyEventSequence(const vector<KeyEvent> &events) const 564 { 565 vector<int32_t> downKeys; 566 for (auto &event : events) { 567 if (event.stage_ == ActionStage::UP) { 568 auto iter = std::find(downKeys.begin(), downKeys.end(), event.code_); 569 if (iter == downKeys.end()) { 570 LOG_W("Cannot release a not-pressed key: %{public}d", event.code_); 571 continue; 572 } 573 downKeys.erase(iter); 574 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 575 keyEvent->SetKeyCode(event.code_); 576 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_UP); 577 OHOS::MMI::KeyEvent::KeyItem keyItem; 578 keyItem.SetKeyCode(event.code_); 579 keyItem.SetPressed(true); 580 keyEvent->AddKeyItem(keyItem); 581 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 582 } else { 583 downKeys.push_back(event.code_); 584 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 585 for (auto downKey : downKeys) { 586 keyEvent->SetKeyCode(downKey); 587 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_DOWN); 588 OHOS::MMI::KeyEvent::KeyItem keyItem; 589 keyItem.SetKeyCode(downKey); 590 keyItem.SetPressed(true); 591 keyEvent->AddKeyItem(keyItem); 592 } 593 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 594 if (event.holdMs_ > 0) { 595 this_thread::sleep_for(chrono::milliseconds(event.holdMs_)); 596 } 597 } 598 } 599 // check not released keys 600 for (auto downKey : downKeys) { 601 LOG_W("Key event sequence injections done with not-released key: %{public}d", downKey); 602 } 603 } 604 PutTextToClipboard(string_view text) const605 void SysUiController::PutTextToClipboard(string_view text) const {} 606 IsWorkable() const607 bool SysUiController::IsWorkable() const 608 { 609 return connected_; 610 } 611 GetCharKeyCode(char ch,int32_t & code,int32_t & ctrlCode) const612 bool SysUiController::GetCharKeyCode(char ch, int32_t &code, int32_t &ctrlCode) const 613 { 614 ctrlCode = KEYCODE_NONE; 615 if (ch >= 'a' && ch <= 'z') { 616 code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'a'); 617 } else if (ch >= 'A' && ch <= 'Z') { 618 ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT; 619 code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'A'); 620 } else if (ch >= '0' && ch <= '9') { 621 code = OHOS::MMI::KeyEvent::KEYCODE_0 + static_cast<int32_t>(ch - '0'); 622 } else { 623 return false; 624 } 625 return true; 626 } 627 TakeScreenCap(int32_t fd,std::stringstream & errReceiver,Rect rect) const628 bool SysUiController::TakeScreenCap(int32_t fd, std::stringstream &errReceiver, Rect rect) const 629 { 630 DisplayManager &displayMgr = DisplayManager::GetInstance(); 631 // get PixelMap from DisplayManager API 632 shared_ptr<PixelMap> pixelMap; 633 if (rect.GetWidth() == 0) { 634 pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId()); 635 } else { 636 Media::Rect region = {.left = rect.left_, .top = rect.top_, 637 .width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_}; 638 Media::Size size = {.width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_}; 639 pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId(), region, size, 0); 640 } 641 if (pixelMap == nullptr) { 642 errReceiver << "Failed to get display pixelMap"; 643 return false; 644 } 645 FILE *fp = fdopen(fd, "wb"); 646 if (fp == nullptr) { 647 perror("File opening failed"); 648 errReceiver << "File opening failed"; 649 return false; 650 } 651 png_structp pngStruct = png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); 652 if (pngStruct == nullptr) { 653 fclose(fp); 654 return false; 655 } 656 png_infop pngInfo = png_create_info_struct(pngStruct); 657 if (pngInfo == nullptr) { 658 fclose(fp); 659 png_destroy_write_struct(&pngStruct, nullptr); 660 return false; 661 } 662 png_init_io(pngStruct, fp); 663 auto width = static_cast<uint32_t>(pixelMap->GetWidth()); 664 auto height = static_cast<uint32_t>(pixelMap->GetHeight()); 665 auto data = pixelMap->GetPixels(); 666 auto stride = static_cast<uint32_t>(pixelMap->GetRowBytes()); 667 // set png header 668 static constexpr int bitmapDepth = 8; 669 png_set_IHDR(pngStruct, pngInfo, width, height, bitmapDepth, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, 670 PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE); 671 png_set_packing(pngStruct); // set packing info 672 png_write_info(pngStruct, pngInfo); // write to header 673 for (uint32_t column = 0; column < height; column++) { 674 png_write_row(pngStruct, data + (column * stride)); 675 } 676 // free/close 677 png_write_end(pngStruct, pngInfo); 678 png_destroy_write_struct(&pngStruct, &pngInfo); 679 (void)fclose(fp); 680 return true; 681 } 682 ConnectToSysAbility()683 bool SysUiController::ConnectToSysAbility() 684 { 685 if (connected_) { 686 return true; 687 } 688 mutex mtx; 689 unique_lock<mutex> uLock(mtx); 690 condition_variable condition; 691 auto onConnectCallback = [&condition]() { 692 LOG_I("Success connect to AccessibilityUITestAbility"); 693 condition.notify_all(); 694 }; 695 auto onDisConnectCallback = [this]() { this->connected_ = false; }; 696 if (g_monitorInstance_ == nullptr) { 697 g_monitorInstance_ = make_shared<UiEventMonitor>(); 698 } 699 g_monitorInstance_->SetOnAbilityConnectCallback(onConnectCallback); 700 g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback); 701 auto ability = AccessibilityUITestAbility::GetInstance(); 702 if (ability->RegisterAbilityListener(g_monitorInstance_) != RET_OK) { 703 LOG_E("Failed to register UiEventMonitor"); 704 return false; 705 } 706 auto ret = ability->Connect(); 707 switch (ret) { 708 case (RET_ERR_INVALID_PARAM): 709 LOG_E("Failed to connect to AccessibilityUITestAbility, RET_ERR_INVALID_PARAM"); 710 return false; 711 case (RET_ERR_NULLPTR): 712 LOG_E("Failed to connect to AccessibilityUITestAbility, RET_ERR_NULLPTR"); 713 return false; 714 case (RET_ERR_CONNECTION_EXIST): 715 LOG_E("Failed to connect to AccessibilityUITestAbility, RET_ERR_CONNECTION_EXIST"); 716 return false; 717 case (RET_ERR_IPC_FAILED): 718 LOG_E("Failed to connect to AccessibilityUITestAbility, RET_ERR_IPC_FAILED"); 719 return false; 720 case (RET_ERR_SAMGR): 721 LOG_E("Failed to connect to AccessibilityUITestAbility, RET_ERR_SAMGR"); 722 return false; 723 default: 724 break; 725 } 726 const auto timeout = chrono::milliseconds(1000); 727 if (condition.wait_for(uLock, timeout) == cv_status::timeout) { 728 LOG_E("Wait connection to AccessibilityUITestAbility timed out"); 729 return false; 730 } 731 connected_ = true; 732 return true; 733 } 734 RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const735 void SysUiController::RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const 736 { 737 g_monitorInstance_->RegisterUiEventListener(listener); 738 } 739 WaitForUiSteady(uint32_t idleThresholdMs,uint32_t timeoutMs) const740 bool SysUiController::WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutMs) const 741 { 742 return g_monitorInstance_->WaitEventIdle(idleThresholdMs, timeoutMs); 743 } 744 DisConnectFromSysAbility()745 void SysUiController::DisConnectFromSysAbility() 746 { 747 if (!connected_ || g_monitorInstance_ == nullptr) { 748 return; 749 } 750 connected_ = false; 751 mutex mtx; 752 unique_lock<mutex> uLock(mtx); 753 condition_variable condition; 754 auto onDisConnectCallback = [&condition]() { 755 LOG_I("Success disconnect from AccessibilityUITestAbility"); 756 condition.notify_all(); 757 }; 758 g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback); 759 auto ability = AccessibilityUITestAbility::GetInstance(); 760 LOG_I("Start disconnect from AccessibilityUITestAbility"); 761 if (ability->Disconnect() != RET_OK) { 762 LOG_E("Failed to disconnect from AccessibilityUITestAbility"); 763 return; 764 } 765 const auto timeout = chrono::milliseconds(200); 766 if (condition.wait_for(uLock, timeout) == cv_status::timeout) { 767 LOG_E("Wait disconnection from AccessibilityUITestAbility timed out"); 768 return; 769 } 770 } 771 SetDisplayRotation(DisplayRotation rotation) const772 void SysUiController::SetDisplayRotation(DisplayRotation rotation) const 773 { 774 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 775 if (display == nullptr) { 776 LOG_E("DisplayManager init fail"); 777 return; 778 } 779 auto screenId = display->GetScreenId(); 780 ScreenManager &screenMgr = ScreenManager::GetInstance(); 781 DCHECK(screenMgr); 782 auto screen = screenMgr.GetScreenById(screenId); 783 if (screen == nullptr) { 784 LOG_E("ScreenManager init fail"); 785 return; 786 } 787 switch (rotation) { 788 case ROTATION_0 : 789 screen->SetOrientation(Orientation::VERTICAL); 790 break; 791 case ROTATION_90 : 792 screen->SetOrientation(Orientation::HORIZONTAL); 793 break; 794 case ROTATION_180 : 795 screen->SetOrientation(Orientation::REVERSE_VERTICAL); 796 break; 797 case ROTATION_270 : 798 screen->SetOrientation(Orientation::REVERSE_HORIZONTAL); 799 break; 800 default : 801 break; 802 } 803 } 804 GetDisplayRotation() const805 DisplayRotation SysUiController::GetDisplayRotation() const 806 { 807 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 808 if (display == nullptr) { 809 LOG_E("DisplayManager init fail"); 810 return DisplayRotation::ROTATION_0; 811 } 812 auto rotation = (DisplayRotation)display->GetRotation(); 813 return rotation; 814 } 815 SetDisplayRotationEnabled(bool enabled) const816 void SysUiController::SetDisplayRotationEnabled(bool enabled) const 817 { 818 ScreenManager &screenMgr = ScreenManager::GetInstance(); 819 DCHECK(screenMgr); 820 screenMgr.SetScreenRotationLocked(enabled); 821 } 822 GetDisplaySize() const823 Point SysUiController::GetDisplaySize() const 824 { 825 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 826 if (display == nullptr) { 827 LOG_E("DisplayManager init fail"); 828 return {0, 0}; 829 } 830 auto width = display->GetWidth(); 831 auto height = display->GetHeight(); 832 Point result(width, height); 833 return result; 834 } 835 GetDisplayDensity() const836 Point SysUiController::GetDisplayDensity() const 837 { 838 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 839 if (display == nullptr) { 840 LOG_E("DisplayManager init fail"); 841 return {0, 0}; 842 } 843 auto rate = display->GetVirtualPixelRatio(); 844 Point displaySize = GetDisplaySize(); 845 Point result(displaySize.px_ * rate, displaySize.py_ * rate); 846 return result; 847 } 848 IsScreenOn() const849 bool SysUiController::IsScreenOn() const 850 { 851 DisplayManager &displayMgr = DisplayManager::GetInstance(); 852 DCHECK(displayMgr); 853 auto displayId = displayMgr.GetDefaultDisplayId(); 854 auto state = displayMgr.GetDisplayState(displayId); 855 return (state != DisplayState::OFF); 856 } 857 } // namespace OHOS::uitest