1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #include <atomic> 17 #include <chrono> 18 #include <fstream> 19 #include <memory> 20 #include <iostream> 21 #include <thread> 22 #include <utility> 23 #include <condition_variable> 24 #include <sys/mman.h> 25 #ifdef HIDUMPER_ENABLED 26 #include <iservice_registry.h> 27 #include <system_ability_load_callback_stub.h> 28 #include "idump_broker.h" 29 #include "dump_broker_proxy.h" 30 #include "system_ability_definition.h" 31 #endif 32 #include "pasteboard_client.h" 33 #include "accessibility_event_info.h" 34 #include "accessibility_ui_test_ability.h" 35 #include "ability_manager_client.h" 36 #include "display_manager.h" 37 #include "screen_manager.h" 38 #include "input_manager.h" 39 #include "png.h" 40 #include "wm_common.h" 41 #include "element_node_iterator_impl.h" 42 #include "system_ui_controller.h" 43 #include "test_server_client.h" 44 45 using namespace std; 46 using namespace chrono; 47 48 namespace OHOS::uitest { 49 using namespace std; 50 using namespace nlohmann; 51 using namespace OHOS::MMI; 52 using namespace OHOS::Accessibility; 53 using namespace OHOS::Rosen; 54 using namespace OHOS::Media; 55 using namespace OHOS::HiviewDFX; 56 using namespace OHOS; 57 58 class UiEventMonitor final : public AccessibleAbilityListener { 59 public: 60 virtual ~UiEventMonitor() override = default; 61 62 void OnAbilityConnected() override; 63 64 void OnAbilityDisconnected() override; 65 66 void OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) override; 67 68 void SetOnAbilityConnectCallback(function<void()> onConnectCb); 69 70 void SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb); 71 OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> & keyEvent)72 bool OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> &keyEvent) override 73 { 74 return false; 75 } 76 77 uint64_t GetLastEventMillis(); 78 79 bool WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs); 80 81 void WaitScrollCompelete(); 82 83 void RegisterUiEventListener(shared_ptr<UiEventListener> listerner); 84 85 private: 86 function<void()> onConnectCallback_ = nullptr; 87 function<void()> onDisConnectCallback_ = nullptr; 88 atomic<uint64_t> lastEventMillis_ = 0; 89 atomic<uint64_t> lastScrollBeginEventMillis_ = 0; 90 atomic<bool> scrollCompelete_ = true; 91 vector<shared_ptr<UiEventListener>> listeners_; 92 }; 93 94 struct EventSpec { 95 std::string_view componentTyep; 96 int32_t eventType; 97 std::string_view event; 98 }; 99 100 const std::map<char, int32_t> SingleKeySymbalMap = { 101 {' ', OHOS::MMI::KeyEvent::KEYCODE_SPACE}, 102 {'`', OHOS::MMI::KeyEvent::KEYCODE_GRAVE}, 103 {'[', OHOS::MMI::KeyEvent::KEYCODE_LEFT_BRACKET}, 104 {']', OHOS::MMI::KeyEvent::KEYCODE_RIGHT_BRACKET}, 105 {'\\', OHOS::MMI::KeyEvent::KEYCODE_BACKSLASH}, 106 {',', OHOS::MMI::KeyEvent::KEYCODE_COMMA}, 107 {';', OHOS::MMI::KeyEvent::KEYCODE_SEMICOLON}, 108 {'\'', OHOS::MMI::KeyEvent::KEYCODE_APOSTROPHE}, 109 {'/', OHOS::MMI::KeyEvent::KEYCODE_SLASH}, 110 {'*', OHOS::MMI::KeyEvent::KEYCODE_NUMPAD_MULTIPLY}, 111 {'-', OHOS::MMI::KeyEvent::KEYCODE_MINUS}, 112 {'.', OHOS::MMI::KeyEvent::KEYCODE_PERIOD}, 113 {'=', OHOS::MMI::KeyEvent::KEYCODE_EQUALS} 114 }; 115 116 const std::map<char, int32_t> MultiKeySymbalMap = { 117 {'~', OHOS::MMI::KeyEvent::KEYCODE_GRAVE}, 118 {'!', OHOS::MMI::KeyEvent::KEYCODE_1}, 119 {'@', OHOS::MMI::KeyEvent::KEYCODE_2}, 120 {'#', OHOS::MMI::KeyEvent::KEYCODE_3}, 121 {'$', OHOS::MMI::KeyEvent::KEYCODE_4}, 122 {'%', OHOS::MMI::KeyEvent::KEYCODE_5}, 123 {'^', OHOS::MMI::KeyEvent::KEYCODE_6}, 124 {'&', OHOS::MMI::KeyEvent::KEYCODE_7}, 125 {'(', OHOS::MMI::KeyEvent::KEYCODE_9}, 126 {')', OHOS::MMI::KeyEvent::KEYCODE_0}, 127 {'+', OHOS::MMI::KeyEvent::KEYCODE_EQUALS}, 128 {'_', OHOS::MMI::KeyEvent::KEYCODE_MINUS}, 129 {':', OHOS::MMI::KeyEvent::KEYCODE_SEMICOLON}, 130 {'"', OHOS::MMI::KeyEvent::KEYCODE_APOSTROPHE}, 131 {'<', OHOS::MMI::KeyEvent::KEYCODE_COMMA}, 132 {'>', OHOS::MMI::KeyEvent::KEYCODE_PERIOD}, 133 {'?', OHOS::MMI::KeyEvent::KEYCODE_SLASH}, 134 {'{', OHOS::MMI::KeyEvent::KEYCODE_LEFT_BRACKET}, 135 {'}', OHOS::MMI::KeyEvent::KEYCODE_RIGHT_BRACKET}, 136 {'|', OHOS::MMI::KeyEvent::KEYCODE_BACKSLASH} 137 }; 138 139 static constexpr EventSpec WATCHED_EVENTS[] = { 140 {"Toast", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "toastShow"}, 141 {"AlertDialog", WindowsContentChangeTypes::CONTENT_CHANGE_TYPE_SUBTREE, "dialogShow"} 142 }; 143 GetWatchedEvent(const AccessibilityEventInfo & eventInfo)144 static std::string GetWatchedEvent(const AccessibilityEventInfo &eventInfo) 145 { 146 auto eventCounts = sizeof(WATCHED_EVENTS) / sizeof(EventSpec); 147 for (unsigned long index = 0; index < eventCounts; index++) { 148 if (WATCHED_EVENTS[index].componentTyep == eventInfo.GetComponentType() && 149 WATCHED_EVENTS[index].eventType == eventInfo.GetWindowContentChangeTypes()) { 150 LOG_W("Capture event: %{public}s", WATCHED_EVENTS[index].event.data()); 151 return string(WATCHED_EVENTS[index].event); 152 } 153 } 154 return "undefine"; 155 } 156 157 // UiEventMonitor instance. 158 static shared_ptr<UiEventMonitor> g_monitorInstance_ = make_shared<UiEventMonitor>(); 159 SetOnAbilityConnectCallback(function<void ()> onConnectCb)160 void UiEventMonitor::SetOnAbilityConnectCallback(function<void()> onConnectCb) 161 { 162 onConnectCallback_ = std::move(onConnectCb); 163 } 164 SetOnAbilityDisConnectCallback(function<void ()> onDisConnectCb)165 void UiEventMonitor::SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb) 166 { 167 onDisConnectCallback_ = std::move(onDisConnectCb); 168 } 169 OnAbilityConnected()170 void UiEventMonitor::OnAbilityConnected() 171 { 172 if (onConnectCallback_ != nullptr) { 173 onConnectCallback_(); 174 } 175 } 176 OnAbilityDisconnected()177 void UiEventMonitor::OnAbilityDisconnected() 178 { 179 if (onDisConnectCallback_ != nullptr) { 180 onDisConnectCallback_(); 181 } 182 } 183 184 // the monitored events 185 static constexpr uint32_t EVENT_MASK = EventType::TYPE_VIEW_TEXT_UPDATE_EVENT | 186 EventType::TYPE_PAGE_STATE_UPDATE | 187 EventType::TYPE_PAGE_CONTENT_UPDATE | 188 EventType::TYPE_VIEW_SCROLLED_EVENT | 189 EventType::TYPE_WINDOW_UPDATE; 190 RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner)191 void UiEventMonitor::RegisterUiEventListener(std::shared_ptr<UiEventListener> listerner) 192 { 193 listeners_.emplace_back(listerner); 194 } 195 OnAccessibilityEvent(const AccessibilityEventInfo & eventInfo)196 void UiEventMonitor::OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) 197 { 198 auto eventType = eventInfo.GetEventType(); 199 LOG_D("OnEvent:0x%{public}x", eventType); 200 auto capturedEvent = GetWatchedEvent(eventInfo); 201 if (eventType == Accessibility::EventType::TYPE_VIEW_SCROLLED_START) { 202 LOG_D("Capture scroll begin"); 203 scrollCompelete_.store(false); 204 lastScrollBeginEventMillis_.store(GetCurrentMillisecond()); 205 } 206 if (eventType == Accessibility::EventType::TYPE_VIEW_SCROLLED_EVENT) { 207 LOG_D("Capture scroll end"); 208 scrollCompelete_.store(true); 209 } 210 if (capturedEvent != "undefine") { 211 auto bundleName = eventInfo.GetBundleName(); 212 auto contentList = eventInfo.GetContentList(); 213 auto text = !contentList.empty() ? contentList[0] : ""; 214 auto type = eventInfo.GetComponentType(); 215 UiEventSourceInfo uiEventSourceInfo = {bundleName, text, type}; 216 for (auto &listener : listeners_) { 217 listener->OnEvent(capturedEvent, uiEventSourceInfo); 218 } 219 } 220 if ((eventInfo.GetEventType() & EVENT_MASK) > 0) { 221 lastEventMillis_.store(GetCurrentMillisecond()); 222 } 223 } 224 GetLastEventMillis()225 uint64_t UiEventMonitor::GetLastEventMillis() 226 { 227 if (lastEventMillis_.load() <= 0) { 228 lastEventMillis_.store(GetCurrentMillisecond()); 229 } 230 return lastEventMillis_.load(); 231 } 232 WaitScrollCompelete()233 void UiEventMonitor::WaitScrollCompelete() 234 { 235 if (scrollCompelete_.load()) { 236 return; 237 } 238 auto currentMs = GetCurrentMillisecond(); 239 if (lastScrollBeginEventMillis_.load() <= 0) { 240 lastScrollBeginEventMillis_.store(currentMs); 241 } 242 const auto idleThresholdMs = 10000; 243 static constexpr auto sliceMs = 10; 244 while (currentMs - lastScrollBeginEventMillis_.load() < idleThresholdMs) { 245 if (scrollCompelete_.load()) { 246 return; 247 } 248 this_thread::sleep_for(chrono::milliseconds(sliceMs)); 249 currentMs = GetCurrentMillisecond(); 250 } 251 LOG_E("wait for scrollEnd event timeout."); 252 scrollCompelete_.store(true); 253 return; 254 } 255 WaitEventIdle(uint32_t idleThresholdMs,uint32_t timeoutMs)256 bool UiEventMonitor::WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs) 257 { 258 const auto currentMs = GetCurrentMillisecond(); 259 if (lastEventMillis_.load() <= 0) { 260 lastEventMillis_.store(currentMs); 261 } 262 if (currentMs - lastEventMillis_.load() >= idleThresholdMs) { 263 return true; 264 } 265 static constexpr auto sliceMs = 10; 266 this_thread::sleep_for(chrono::milliseconds(sliceMs)); 267 if (timeoutMs <= sliceMs) { 268 return false; 269 } 270 return WaitEventIdle(idleThresholdMs, timeoutMs - sliceMs); 271 } 272 SysUiController()273 SysUiController::SysUiController() : UiController() {} 274 ~SysUiController()275 SysUiController::~SysUiController() 276 { 277 DisConnectFromSysAbility(); 278 } 279 Initialize(ApiCallErr & error)280 bool SysUiController::Initialize(ApiCallErr &error) 281 { 282 return this->ConnectToSysAbility(error); 283 } 284 GetVisibleRect(Rect windowBounds,Accessibility::Rect nodeBounds)285 static Rect GetVisibleRect(Rect windowBounds, Accessibility::Rect nodeBounds) 286 { 287 auto leftX = nodeBounds.GetLeftTopXScreenPostion(); 288 auto topY = nodeBounds.GetLeftTopYScreenPostion(); 289 auto rightX = nodeBounds.GetRightBottomXScreenPostion(); 290 auto bottomY = nodeBounds.GetRightBottomYScreenPostion(); 291 Rect newBounds((leftX < windowBounds.left_) ? windowBounds.left_ : leftX, 292 (rightX > windowBounds.right_) ? windowBounds.right_ : rightX, 293 (topY < windowBounds.top_) ? windowBounds.top_ : topY, 294 (bottomY > windowBounds.bottom_) ? windowBounds.bottom_ : bottomY); 295 return newBounds; 296 } 297 InflateWindowInfo(AccessibilityWindowInfo & node,Window & info)298 static void InflateWindowInfo(AccessibilityWindowInfo& node, Window& info) 299 { 300 info.focused_ = node.IsFocused(); 301 info.actived_ = node.IsActive(); 302 info.decoratorEnabled_ = node.IsDecorEnable(); 303 // get bundle name by root node 304 AccessibilityElementInfo element; 305 LOG_D("Start Get Bundle Name by WindowId %{public}d", node.GetWindowId()); 306 if (AccessibilityUITestAbility::GetInstance()->GetRootByWindow(node, element) != RET_OK) { 307 LOG_E("Failed Get Bundle Name by WindowId %{public}d", node.GetWindowId()); 308 } else { 309 std::string app = element.GetBundleName(); 310 LOG_I("End Get Bundle Name by WindowId %{public}d, app is %{public}s", node.GetWindowId(), app.data()); 311 info.bundleName_ = app; 312 const auto foreAbility = AAFwk::AbilityManagerClient::GetInstance()->GetTopAbility(); 313 info.abilityName_ = (app == foreAbility.GetBundleName()) ? foreAbility.GetAbilityName() : ""; 314 info.pagePath_ = (app == foreAbility.GetBundleName()) ? element.GetPagePath() : ""; 315 } 316 info.mode_ = WindowMode::UNKNOWN; 317 const auto origMode = static_cast<OHOS::Rosen::WindowMode>(node.GetWindowMode()); 318 switch (origMode) { 319 case OHOS::Rosen::WindowMode::WINDOW_MODE_FULLSCREEN: 320 info.mode_ = WindowMode::FULLSCREEN; 321 break; 322 case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_PRIMARY: 323 info.mode_ = WindowMode::SPLIT_PRIMARY; 324 break; 325 case OHOS::Rosen::WindowMode::WINDOW_MODE_SPLIT_SECONDARY: 326 info.mode_ = WindowMode::SPLIT_SECONDARY; 327 break; 328 case OHOS::Rosen::WindowMode::WINDOW_MODE_FLOATING: 329 info.mode_ = WindowMode::FLOATING; 330 break; 331 case OHOS::Rosen::WindowMode::WINDOW_MODE_PIP: 332 info.mode_ = WindowMode::PIP; 333 break; 334 default: 335 info.mode_ = WindowMode::UNKNOWN; 336 break; 337 } 338 } 339 GetAamsWindowInfos(vector<AccessibilityWindowInfo> & windows)340 static bool GetAamsWindowInfos(vector<AccessibilityWindowInfo> &windows) 341 { 342 auto ability = AccessibilityUITestAbility::GetInstance(); 343 g_monitorInstance_->WaitScrollCompelete(); 344 if (ability->GetWindows(windows) != RET_OK) { 345 LOG_W("GetWindows from AccessibilityUITestAbility failed"); 346 return false; 347 } 348 sort(windows.begin(), windows.end(), [](auto &w1, auto &w2) -> bool { 349 return w1.GetWindowLayer() > w2.GetWindowLayer(); 350 }); 351 return true; 352 } 353 GetUiWindows(std::vector<Window> & out)354 void SysUiController::GetUiWindows(std::vector<Window> &out) 355 { 356 std::lock_guard<std::mutex> dumpLocker(dumpMtx); // disallow concurrent dumpUi 357 ApiCallErr error = ApiCallErr(NO_ERROR); 358 if (!connected_ && !ConnectToSysAbility(error)) { 359 LOG_E("%{public}s", error.message_.c_str()); 360 return; 361 } 362 vector<AccessibilityWindowInfo> windows; 363 LOG_D("Get Window root info"); 364 if (!GetAamsWindowInfos(windows)) { 365 return; 366 } 367 LOG_D("End Get Window root info"); 368 auto screenSize = GetDisplaySize(); 369 auto screenRect = Rect(0, screenSize.px_, 0, screenSize.py_); 370 std::vector<Rect> overplays; 371 // window wrapper 372 for (auto &win : windows) { 373 Rect winRectInScreen = GetVisibleRect(screenRect, win.GetRectInScreen()); 374 Rect visibleArea = winRectInScreen; 375 if (!RectAlgorithm::ComputeMaxVisibleRegion(winRectInScreen, overplays, visibleArea)) { 376 LOG_I("window is covered, windowId : %{public}d, layer is %{public}d", win.GetWindowId(), 377 win.GetWindowLayer()); 378 continue; 379 } 380 LOG_I("window is visible, windowId: %{public}d, active: %{public}d, focus: %{public}d, layer: %{public}d", 381 win.GetWindowId(), win.IsActive(), win.IsFocused(), win.GetWindowLayer()); 382 Window winWrapper{win.GetWindowId()}; 383 InflateWindowInfo(win, winWrapper); 384 winWrapper.bounds_ = winRectInScreen; 385 for (const auto &overWin : overplays) { 386 Rect intersectionRect{0, 0, 0, 0}; 387 if (RectAlgorithm::ComputeIntersection(winRectInScreen, overWin, intersectionRect)) { 388 winWrapper.invisibleBoundsVec_.emplace_back(overWin); 389 } 390 } 391 RectAlgorithm::ComputeMaxVisibleRegion(winWrapper.bounds_, overplays, winWrapper.visibleBounds_); 392 overplays.emplace_back(winRectInScreen); 393 out.emplace_back(std::move(winWrapper)); 394 } 395 } 396 GetWidgetsInWindow(const Window & winInfo,unique_ptr<ElementNodeIterator> & elementIterator)397 bool SysUiController::GetWidgetsInWindow(const Window &winInfo, unique_ptr<ElementNodeIterator> &elementIterator) 398 { 399 std::lock_guard<std::mutex> dumpLocker(dumpMtx); // disallow concurrent dumpUi 400 if (!connected_) { 401 LOG_W("Connect to AccessibilityUITestAbility failed"); 402 return false; 403 } 404 std::vector<AccessibilityElementInfo> elementInfos; 405 AccessibilityWindowInfo window; 406 LOG_D("Get Window by WindowId %{public}d", winInfo.id_); 407 if (AccessibilityUITestAbility::GetInstance()->GetWindow(winInfo.id_, window) != RET_OK) { 408 LOG_E("GetWindowInfo failed, windowId: %{public}d", winInfo.id_); 409 return false; 410 } 411 LOG_D("Start Get nodes from window by WindowId %{public}d", winInfo.id_); 412 if (AccessibilityUITestAbility::GetInstance()->GetRootByWindowBatch(window, elementInfos) != RET_OK) { 413 LOG_E("GetRootByWindowBatch failed, windowId: %{public}d", winInfo.id_); 414 return false; 415 } else { 416 LOG_I("End Get nodes from window by WindowId %{public}d, node size is %{public}zu, appId: %{public}s", 417 winInfo.id_, elementInfos.size(), winInfo.bundleName_.data()); 418 elementIterator = std::make_unique<ElementNodeIteratorImpl>(elementInfos); 419 } 420 return true; 421 } 422 AddPinterItems(PointerEvent & event,const vector<pair<bool,Point>> & fingerStatus,uint32_t currentFinger)423 static void AddPinterItems(PointerEvent &event, const vector<pair<bool, Point>> &fingerStatus, 424 uint32_t currentFinger) 425 { 426 PointerEvent::PointerItem pinterItem1; 427 pinterItem1.SetPointerId(currentFinger); 428 pinterItem1.SetOriginPointerId(currentFinger); 429 pinterItem1.SetDisplayX(fingerStatus[currentFinger].second.px_); 430 pinterItem1.SetDisplayY(fingerStatus[currentFinger].second.py_); 431 pinterItem1.SetRawDisplayX(fingerStatus[currentFinger].second.px_); 432 pinterItem1.SetRawDisplayY(fingerStatus[currentFinger].second.py_); 433 pinterItem1.SetPressed(fingerStatus[currentFinger].first); 434 event.UpdatePointerItem(currentFinger, pinterItem1); 435 // update pinterItem of other fingers which in pressed state. 436 for (uint32_t index = 0; index < fingerStatus.size(); index++) { 437 if (index == currentFinger) { 438 continue; 439 } 440 if (fingerStatus[index].first) { 441 PointerEvent::PointerItem pinterItem; 442 pinterItem.SetPointerId(index); 443 pinterItem.SetOriginPointerId(index); 444 pinterItem.SetDisplayX(fingerStatus[index].second.px_); 445 pinterItem.SetDisplayY(fingerStatus[index].second.py_); 446 pinterItem.SetRawDisplayX(fingerStatus[index].second.px_); 447 pinterItem.SetRawDisplayY(fingerStatus[index].second.py_); 448 pinterItem.SetPressed(true); 449 event.UpdatePointerItem(index, pinterItem); 450 } 451 } 452 } 453 InjectTouchEventSequence(const PointerMatrix & events) const454 void SysUiController::InjectTouchEventSequence(const PointerMatrix &events) const 455 { 456 // fingerStatus stores the press status and coordinates of each finger. 457 vector<pair<bool, Point>> fingerStatus(events.GetFingers(), make_pair(false, Point(0,0))); 458 for (uint32_t step = 0; step < events.GetSteps(); step++) { 459 auto pointerEvent = PointerEvent::Create(); 460 if (pointerEvent == nullptr) { 461 LOG_E("Creat PointerEvent failed."); 462 return; 463 } 464 for (uint32_t finger = 0; finger < events.GetFingers(); finger++) { 465 bool isPressed = events.At(finger, step).stage_ != ActionStage::UP; 466 fingerStatus[finger] = make_pair(isPressed, events.At(finger, step).point_); 467 pointerEvent->SetPointerId(finger); 468 switch (events.At(finger, step).stage_) { 469 case ActionStage::DOWN: 470 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_DOWN); 471 break; 472 case ActionStage::MOVE: 473 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_MOVE); 474 break; 475 case ActionStage::UP: 476 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_UP); 477 break; 478 default: 479 break; 480 } 481 AddPinterItems(*pointerEvent, fingerStatus, finger); 482 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_TOUCHSCREEN); 483 DisplayManager &displayMgr = DisplayManager::GetInstance(); 484 pointerEvent->SetTargetDisplayId(displayMgr.GetDefaultDisplayId()); 485 InputManager::GetInstance()->SimulateInputEvent(pointerEvent); 486 if (events.At(finger, step).holdMs_ > 0) { 487 this_thread::sleep_for(chrono::milliseconds(events.At(finger, step).holdMs_)); 488 } 489 } 490 } 491 } 492 SetMousePointerItemAttr(const MouseEvent & event,PointerEvent::PointerItem & item)493 static void SetMousePointerItemAttr(const MouseEvent &event, PointerEvent::PointerItem &item) 494 { 495 item.SetPointerId(0); 496 item.SetOriginPointerId(0); 497 item.SetToolType(PointerEvent::TOOL_TYPE_MOUSE); 498 item.SetDisplayX(event.point_.px_); 499 item.SetDisplayY(event.point_.py_); 500 item.SetRawDisplayX(event.point_.px_); 501 item.SetRawDisplayY(event.point_.py_); 502 item.SetPressed(false); 503 item.SetDownTime(0); 504 LOG_D("Inject mouseEvent, pressed:%{public}d, location:%{public}d, %{public}d", 505 event.stage_ == ActionStage::DOWN, event.point_.px_, event.point_.py_); 506 } 507 SetMousePointerEventAttr(shared_ptr<PointerEvent> pointerEvent,const MouseEvent & event)508 static void SetMousePointerEventAttr(shared_ptr<PointerEvent> pointerEvent, const MouseEvent &event) 509 { 510 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_MOUSE); 511 pointerEvent->SetPointerId(0); 512 if (event.button_ != MouseButton::BUTTON_NONE) { 513 pointerEvent->SetButtonId(event.button_); 514 if ((event.stage_ == ActionStage::DOWN || event.stage_ == ActionStage::MOVE)) { 515 pointerEvent->SetButtonPressed(event.button_); 516 } else if (event.stage_ == ActionStage::UP) { 517 pointerEvent->DeleteReleaseButton(event.button_); 518 } 519 } 520 } 521 InjectMouseEvent(const MouseEvent & event) const522 void SysUiController::InjectMouseEvent(const MouseEvent &event) const 523 { 524 auto pointerEvent = PointerEvent::Create(); 525 if (pointerEvent == nullptr) { 526 return; 527 } 528 PointerEvent::PointerItem item; 529 SetMousePointerEventAttr(pointerEvent, event); 530 constexpr double axialValue = 15; 531 static bool flag = true; 532 auto injectAxialValue = axialValue; 533 switch (event.stage_) { 534 case ActionStage::DOWN: 535 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_DOWN); 536 item.SetPressed(true); 537 break; 538 case ActionStage::MOVE: 539 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_MOVE); 540 break; 541 case ActionStage::UP: 542 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_BUTTON_UP); 543 break; 544 case ActionStage::AXIS_UP: 545 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_BEGIN); 546 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, -axialValue); 547 flag = false; 548 break; 549 case ActionStage::AXIS_DOWN: 550 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_BEGIN); 551 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, axialValue); 552 flag = true; 553 break; 554 case ActionStage::AXIS_STOP: 555 pointerEvent->SetPointerAction(OHOS::MMI::PointerEvent::POINTER_ACTION_AXIS_END); 556 injectAxialValue = flag ? axialValue : -axialValue; 557 pointerEvent->SetAxisValue(OHOS::MMI::PointerEvent::AXIS_TYPE_SCROLL_VERTICAL, injectAxialValue); 558 break; 559 default: 560 break; 561 } 562 SetMousePointerItemAttr(event, item); 563 pointerEvent->AddPointerItem(item); 564 if (!downKeys_.empty()) { 565 pointerEvent->SetPressedKeys(downKeys_); 566 } 567 InputManager::GetInstance()->SimulateInputEvent(pointerEvent); 568 this_thread::sleep_for(chrono::milliseconds(event.holdMs_)); 569 } 570 InjectMouseEventSequence(const vector<MouseEvent> & events) const571 void SysUiController::InjectMouseEventSequence(const vector<MouseEvent> &events) const 572 { 573 for (auto &event : events) { 574 auto keyEvents = event.keyEvents_; 575 if (!keyEvents.empty() && keyEvents.front().stage_ == ActionStage::DOWN) { 576 InjectKeyEventSequence(keyEvents); 577 InjectMouseEvent(event); 578 } else { 579 InjectMouseEvent(event); 580 InjectKeyEventSequence(keyEvents); 581 } 582 } 583 } 584 InjectKeyEventSequence(const vector<KeyEvent> & events) const585 void SysUiController::InjectKeyEventSequence(const vector<KeyEvent> &events) const 586 { 587 for (auto &event : events) { 588 if (event.code_ == KEYCODE_NONE) { 589 continue; 590 } 591 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 592 if (keyEvent == nullptr) { 593 LOG_E("Creat KeyEvent failed."); 594 return; 595 } 596 if (event.stage_ == ActionStage::UP) { 597 auto iter = std::find(downKeys_.begin(), downKeys_.end(), event.code_); 598 if (iter == downKeys_.end()) { 599 LOG_W("Cannot release a not-pressed key: %{public}d", event.code_); 600 continue; 601 } 602 downKeys_.erase(iter); 603 keyEvent->SetKeyCode(event.code_); 604 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_UP); 605 OHOS::MMI::KeyEvent::KeyItem keyItem; 606 keyItem.SetKeyCode(event.code_); 607 keyItem.SetPressed(false); 608 keyEvent->AddKeyItem(keyItem); 609 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 610 LOG_D("Inject keyEvent up, keycode:%{public}d", event.code_); 611 } else { 612 downKeys_.push_back(event.code_); 613 for (auto downKey : downKeys_) { 614 keyEvent->SetKeyCode(downKey); 615 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_DOWN); 616 OHOS::MMI::KeyEvent::KeyItem keyItem; 617 keyItem.SetKeyCode(downKey); 618 keyItem.SetPressed(true); 619 keyEvent->AddKeyItem(keyItem); 620 } 621 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 622 LOG_D("Inject keyEvent down, keycode:%{public}d", event.code_); 623 if (event.holdMs_ > 0) { 624 this_thread::sleep_for(chrono::milliseconds(event.holdMs_)); 625 } 626 } 627 } 628 // check not released keys 629 for (auto downKey : downKeys_) { 630 LOG_W("Key event sequence injections done with not-released key: %{public}d", downKey); 631 } 632 } 633 PutTextToClipboard(string_view text) const634 void SysUiController::PutTextToClipboard(string_view text) const 635 { 636 OHOS::testserver::TestServerClient::GetInstance().SetPasteData(string(text)); 637 } 638 IsWorkable() const639 bool SysUiController::IsWorkable() const 640 { 641 return connected_; 642 } 643 GetCharKeyCode(char ch,int32_t & code,int32_t & ctrlCode) const644 bool SysUiController::GetCharKeyCode(char ch, int32_t &code, int32_t &ctrlCode) const 645 { 646 ctrlCode = KEYCODE_NONE; 647 if (ch >= 'a' && ch <= 'z') { 648 code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'a'); 649 } else if (ch >= 'A' && ch <= 'Z') { 650 ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT; 651 code = OHOS::MMI::KeyEvent::KEYCODE_A + static_cast<int32_t>(ch - 'A'); 652 } else if (ch >= '0' && ch <= '9') { 653 code = OHOS::MMI::KeyEvent::KEYCODE_0 + static_cast<int32_t>(ch - '0'); 654 } else if (SingleKeySymbalMap.find(ch) != SingleKeySymbalMap.end()) { 655 code = SingleKeySymbalMap.find(ch)->second; 656 } else if (MultiKeySymbalMap.find(ch) != MultiKeySymbalMap.end()) { 657 ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT; 658 code = MultiKeySymbalMap.find(ch)->second; 659 } else { 660 return false; 661 } 662 return true; 663 } 664 TakeScreenCap(int32_t fd,std::stringstream & errReceiver,Rect rect) const665 bool SysUiController::TakeScreenCap(int32_t fd, std::stringstream &errReceiver, Rect rect) const 666 { 667 DisplayManager &displayMgr = DisplayManager::GetInstance(); 668 // get PixelMap from DisplayManager API 669 shared_ptr<PixelMap> pixelMap; 670 if (rect.GetWidth() == 0) { 671 pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId()); 672 } else { 673 Media::Rect region = {.left = rect.left_, .top = rect.top_, 674 .width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_}; 675 Media::Size size = {.width = rect.right_ - rect.left_, .height = rect.bottom_ - rect.top_}; 676 pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId(), region, size, 0); 677 } 678 if (pixelMap == nullptr) { 679 errReceiver << "Failed to get display pixelMap"; 680 return false; 681 } 682 FILE *fp = fdopen(fd, "wb"); 683 if (fp == nullptr) { 684 perror("File opening failed"); 685 errReceiver << "File opening failed"; 686 return false; 687 } 688 png_structp pngStruct = png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); 689 if (pngStruct == nullptr) { 690 fclose(fp); 691 return false; 692 } 693 png_infop pngInfo = png_create_info_struct(pngStruct); 694 if (pngInfo == nullptr) { 695 fclose(fp); 696 png_destroy_write_struct(&pngStruct, nullptr); 697 return false; 698 } 699 png_init_io(pngStruct, fp); 700 auto width = static_cast<uint32_t>(pixelMap->GetWidth()); 701 auto height = static_cast<uint32_t>(pixelMap->GetHeight()); 702 auto data = pixelMap->GetPixels(); 703 auto stride = static_cast<uint32_t>(pixelMap->GetRowBytes()); 704 // set png header 705 static constexpr int bitmapDepth = 8; 706 png_set_IHDR(pngStruct, pngInfo, width, height, bitmapDepth, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, 707 PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE); 708 png_set_packing(pngStruct); // set packing info 709 png_write_info(pngStruct, pngInfo); // write to header 710 for (uint32_t column = 0; column < height; column++) { 711 png_write_row(pngStruct, data + (column * stride)); 712 } 713 // free/close 714 png_write_end(pngStruct, pngInfo); 715 png_destroy_write_struct(&pngStruct, &pngInfo); 716 (void)fclose(fp); 717 return true; 718 } 719 ConnectToSysAbility(ApiCallErr & error)720 bool SysUiController::ConnectToSysAbility(ApiCallErr &error) 721 { 722 if (connected_) { 723 return true; 724 } 725 mutex mtx; 726 unique_lock<mutex> uLock(mtx); 727 std::shared_ptr<condition_variable> condition = make_shared<condition_variable>(); 728 auto onConnectCallback = [condition]() { 729 LOG_I("Success connect to AccessibilityUITestAbility"); 730 condition->notify_all(); 731 }; 732 auto onDisConnectCallback = [this]() { this->connected_ = false; }; 733 if (g_monitorInstance_ == nullptr) { 734 g_monitorInstance_ = make_shared<UiEventMonitor>(); 735 } 736 g_monitorInstance_->SetOnAbilityConnectCallback(onConnectCallback); 737 g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback); 738 auto ability = AccessibilityUITestAbility::GetInstance(); 739 if (ability->RegisterAbilityListener(g_monitorInstance_) != RET_OK) { 740 error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS, REGISTER_LISTENER_FAILED"); 741 return false; 742 } 743 auto ret = ability->Connect(); 744 LOG_I("Connect to AAMS, result: %{public}d", ret); 745 if (ret != RET_OK) { 746 error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS"); 747 if (ret == RET_ERR_CONNECTION_EXIST) { 748 error.message_ += ", RET_ERR_CONNECTION_EXIST"; 749 } else { 750 error.message_ += ", RET_ERR_AAMS"; 751 } 752 return false; 753 } 754 const auto timeout = chrono::milliseconds(3000); 755 if (condition->wait_for(uLock, timeout) == cv_status::timeout) { 756 LOG_E("Wait connection to AccessibilityUITestAbility timed out"); 757 error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS, RET_TIMEOUT"); 758 return false; 759 } 760 connected_ = true; 761 return true; 762 } 763 RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const764 void SysUiController::RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) const 765 { 766 g_monitorInstance_->RegisterUiEventListener(listener); 767 } 768 WaitForUiSteady(uint32_t idleThresholdMs,uint32_t timeoutMs) const769 bool SysUiController::WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutMs) const 770 { 771 return g_monitorInstance_->WaitEventIdle(idleThresholdMs, timeoutMs); 772 } 773 DisConnectFromSysAbility()774 void SysUiController::DisConnectFromSysAbility() 775 { 776 if (!connected_ || g_monitorInstance_ == nullptr) { 777 return; 778 } 779 connected_ = false; 780 mutex mtx; 781 unique_lock<mutex> uLock(mtx); 782 condition_variable condition; 783 auto onDisConnectCallback = [&condition]() { 784 LOG_I("Success disconnect from AccessibilityUITestAbility"); 785 condition.notify_all(); 786 }; 787 g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback); 788 auto ability = AccessibilityUITestAbility::GetInstance(); 789 LOG_I("Start disconnect from AccessibilityUITestAbility"); 790 if (ability->Disconnect() != RET_OK) { 791 LOG_E("Failed to disconnect from AccessibilityUITestAbility"); 792 return; 793 } 794 const auto timeout = chrono::milliseconds(200); 795 if (condition.wait_for(uLock, timeout) == cv_status::timeout) { 796 LOG_E("Wait disconnection from AccessibilityUITestAbility timed out"); 797 return; 798 } 799 } 800 SetDisplayRotation(DisplayRotation rotation) const801 void SysUiController::SetDisplayRotation(DisplayRotation rotation) const 802 { 803 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 804 if (display == nullptr) { 805 LOG_E("DisplayManager init fail"); 806 return; 807 } 808 auto screenId = display->GetScreenId(); 809 ScreenManager &screenMgr = ScreenManager::GetInstance(); 810 DCHECK(screenMgr); 811 bool isLocked = false; 812 screenMgr.IsScreenRotationLocked(isLocked); 813 if (isLocked) { 814 screenMgr.SetScreenRotationLocked(false); 815 } 816 auto screen = screenMgr.GetScreenById(screenId); 817 if (screen == nullptr) { 818 LOG_E("ScreenManager init fail"); 819 return; 820 } 821 switch (rotation) { 822 case ROTATION_0 : 823 screen->SetOrientation(Orientation::VERTICAL); 824 break; 825 case ROTATION_90 : 826 screen->SetOrientation(Orientation::HORIZONTAL); 827 break; 828 case ROTATION_180 : 829 screen->SetOrientation(Orientation::REVERSE_VERTICAL); 830 break; 831 case ROTATION_270 : 832 screen->SetOrientation(Orientation::REVERSE_HORIZONTAL); 833 break; 834 default : 835 break; 836 } 837 } 838 GetDisplayRotation() const839 DisplayRotation SysUiController::GetDisplayRotation() const 840 { 841 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 842 if (display == nullptr) { 843 LOG_E("DisplayManager init fail"); 844 return DisplayRotation::ROTATION_0; 845 } 846 auto rotation = (DisplayRotation)display->GetRotation(); 847 return rotation; 848 } 849 SetDisplayRotationEnabled(bool enabled) const850 void SysUiController::SetDisplayRotationEnabled(bool enabled) const 851 { 852 ScreenManager &screenMgr = ScreenManager::GetInstance(); 853 DCHECK(screenMgr); 854 screenMgr.SetScreenRotationLocked(!enabled); 855 } 856 GetDisplaySize() const857 Point SysUiController::GetDisplaySize() const 858 { 859 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 860 if (display == nullptr) { 861 LOG_E("DisplayManager init fail"); 862 return {0, 0}; 863 } 864 auto width = display->GetWidth(); 865 auto height = display->GetHeight(); 866 LOG_D("GetDisplaysize, width: %{public}d, height: %{public}d", width, height); 867 Point result(width, height); 868 return result; 869 } 870 GetDisplayDensity() const871 Point SysUiController::GetDisplayDensity() const 872 { 873 auto display = DisplayManager::GetInstance().GetDefaultDisplay(); 874 if (display == nullptr) { 875 LOG_E("DisplayManager init fail"); 876 return {0, 0}; 877 } 878 auto rate = display->GetVirtualPixelRatio(); 879 Point displaySize = GetDisplaySize(); 880 Point result(displaySize.px_ * rate, displaySize.py_ * rate); 881 return result; 882 } 883 IsScreenOn() const884 bool SysUiController::IsScreenOn() const 885 { 886 DisplayManager &displayMgr = DisplayManager::GetInstance(); 887 DCHECK(displayMgr); 888 auto displayId = displayMgr.GetDefaultDisplayId(); 889 auto state = displayMgr.GetDisplayState(displayId); 890 return (state != DisplayState::OFF); 891 } 892 893 class OnSaLoadCallback : public SystemAbilityLoadCallbackStub { 894 public: OnSaLoadCallback(mutex & mutex)895 explicit OnSaLoadCallback(mutex &mutex): mutex_(mutex) {}; ~OnSaLoadCallback()896 ~OnSaLoadCallback() {}; OnLoadSystemAbilitySuccess(int32_t systemAbilityId,const sptr<IRemoteObject> & remoteObject)897 void OnLoadSystemAbilitySuccess(int32_t systemAbilityId, const sptr<IRemoteObject>& remoteObject) override 898 { 899 if (systemAbilityId == OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID) { 900 remoteObject_ = remoteObject; 901 mutex_.unlock(); 902 } 903 } OnLoadSystemAbilityFail(int32_t systemAbilityId)904 void OnLoadSystemAbilityFail(int32_t systemAbilityId) override 905 { 906 if (systemAbilityId == OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID) { 907 mutex_.unlock(); 908 } 909 } 910 GetSaObject()911 sptr<IRemoteObject> GetSaObject() 912 { 913 return remoteObject_; 914 } 915 916 private: 917 mutex &mutex_; 918 sptr<IRemoteObject> remoteObject_ = nullptr; 919 }; 920 CreateHidumperCmd(const std::string & windowId,vector<u16string> & result)921 static void CreateHidumperCmd(const std::string &windowId, vector<u16string> &result) 922 { 923 result.emplace_back(u"hidumper"); 924 result.emplace_back(u"-s"); 925 result.emplace_back(u"WindowManagerService"); 926 result.emplace_back(u"-a"); 927 auto winIdInUtf16 = std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>{}.from_bytes(windowId); 928 auto arg = u16string(u"-w ").append(winIdInUtf16).append(u" -default -lastpage"); 929 result.emplace_back(move(arg)); 930 } 931 GetHidumperInfo(std::string windowId,char ** buf,size_t & len)932 void SysUiController::GetHidumperInfo(std::string windowId, char **buf, size_t &len) 933 { 934 #ifdef HIDUMPER_ENABLED 935 auto sam = SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager(); 936 // wati SA start 937 constexpr auto delayMs = 2000; 938 this_thread::sleep_for(chrono::milliseconds(delayMs)); 939 if (sam == nullptr) { 940 LOG_E("Get samgr failed"); 941 return; 942 } 943 auto remoteObject = sam->CheckSystemAbility(OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID); 944 if (remoteObject == nullptr) { 945 mutex lock; 946 lock.lock(); 947 sptr<OnSaLoadCallback> loadCallback = new OnSaLoadCallback(lock); 948 if (sam->LoadSystemAbility(OHOS::DFX_HI_DUMPER_SERVICE_ABILITY_ID, loadCallback) != ERR_OK) { 949 LOG_E("Schedule LoadSystemAbility failed"); 950 lock.unlock(); 951 return; 952 } 953 LOG_E("Schedule LoadSystemAbility succeed"); 954 lock.unlock(); 955 remoteObject = loadCallback->GetSaObject(); 956 LOG_E("LoadSystemAbility callbacked, result = %{public}s", remoteObject == nullptr ? "FAIL" : "SUCCESS"); 957 } 958 if (remoteObject == nullptr) { 959 LOG_E("remoteObject is null"); 960 return; 961 } 962 // run dump command 963 sptr<IDumpBroker> client = iface_cast<IDumpBroker>(remoteObject); 964 if (client == nullptr) { 965 LOG_E("IDumpBroker converts failed"); 966 return; 967 } 968 auto fd = memfd_create("dummy_file", 2); 969 ftruncate(fd, 0); 970 vector<u16string> args; 971 CreateHidumperCmd(windowId, args); 972 client->Request(args, fd); 973 auto size = lseek(fd, 0, SEEK_END); 974 char *tempBuf = new char[size + 1]; 975 lseek(fd, 0, SEEK_SET); 976 read(fd, tempBuf, size); 977 *buf = tempBuf; 978 len = size; 979 close(fd); 980 #else 981 *buf = nullptr; 982 len = 0; 983 #endif 984 } 985 } // namespace OHOS::uitest