1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #include <atomic> 17 #include <chrono> 18 #include <fstream> 19 #include <memory> 20 #include <iostream> 21 #include <thread> 22 #include <utility> 23 #include <condition_variable> 24 #include "accessibility_event_info.h" 25 #include "accessibility_ui_test_ability.h" 26 #include "accessibility_ui_test_ability_listener.h" 27 #include "display_manager.h" 28 #include "input_manager.h" 29 #include "png.h" 30 #include "system_ui_controller.h" 31 32 using namespace std; 33 using namespace chrono; 34 35 namespace OHOS::uitest { 36 using namespace std; 37 using namespace nlohmann; 38 using namespace OHOS::MMI; 39 using namespace OHOS::Accessibility; 40 using namespace OHOS::Rosen; 41 using namespace OHOS::Media; 42 43 class UiEventMonitor final : public IAccessibleUITestAbilityListener { 44 public: 45 virtual ~UiEventMonitor() = default; 46 47 void OnAbilityConnected() override; 48 49 void OnAbilityDisconnected() override; 50 51 void OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) override; 52 53 void SetOnAbilityConnectCallback(function<void()> onConnectCb); 54 55 void SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb); 56 OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> & keyEvent,const int sequence)57 bool OnKeyPressEvent(const shared_ptr<MMI::KeyEvent> &keyEvent, const int sequence) override 58 { 59 return false; 60 } 61 OnGestureSimulateResult(const int sequence,const bool completedSuccessfully)62 void OnGestureSimulateResult(const int sequence, const bool completedSuccessfully) override {} 63 64 uint64_t GetLastEventMillis(); 65 66 bool WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs); 67 68 private: 69 function<void()> onConnectCallback_ = nullptr; 70 function<void()> onDisConnectCallback_ = nullptr; 71 atomic<uint64_t> lastEventMillis_ = 0; 72 }; 73 SetOnAbilityConnectCallback(function<void ()> onConnectCb)74 void UiEventMonitor::SetOnAbilityConnectCallback(function<void()> onConnectCb) 75 { 76 onConnectCallback_ = std::move(onConnectCb); 77 } 78 SetOnAbilityDisConnectCallback(function<void ()> onDisConnectCb)79 void UiEventMonitor::SetOnAbilityDisConnectCallback(function<void()> onDisConnectCb) 80 { 81 onDisConnectCallback_ = std::move(onDisConnectCb); 82 } 83 OnAbilityConnected()84 void UiEventMonitor::OnAbilityConnected() 85 { 86 if (onConnectCallback_ != nullptr) { 87 onConnectCallback_(); 88 } 89 } 90 OnAbilityDisconnected()91 void UiEventMonitor::OnAbilityDisconnected() 92 { 93 if (onDisConnectCallback_ != nullptr) { 94 onDisConnectCallback_(); 95 } 96 } 97 98 // the monitored events 99 static constexpr uint32_t EVENT_MASK = EventType::TYPE_VIEW_TEXT_UPDATE_EVENT 100 | EventType::TYPE_PAGE_STATE_UPDATE | EventType::TYPE_PAGE_CONTENT_UPDATE 101 | EventType::TYPE_VIEW_SCROLLED_EVENT | EventType::TYPE_WINDOW_UPDATE; 102 OnAccessibilityEvent(const AccessibilityEventInfo & eventInfo)103 void UiEventMonitor::OnAccessibilityEvent(const AccessibilityEventInfo &eventInfo) 104 { 105 LOG_W("OnEvent:0x%{public}x", eventInfo.GetEventType()); 106 if ((eventInfo.GetEventType() & EVENT_MASK) > 0) { 107 lastEventMillis_.store(GetCurrentMillisecond()); 108 } 109 } 110 GetLastEventMillis()111 uint64_t UiEventMonitor::GetLastEventMillis() 112 { 113 if (lastEventMillis_.load() <= 0) { 114 lastEventMillis_.store(GetCurrentMillisecond()); 115 } 116 return lastEventMillis_.load(); 117 } 118 WaitEventIdle(uint32_t idleThresholdMs,uint32_t timeoutMs)119 bool UiEventMonitor::WaitEventIdle(uint32_t idleThresholdMs, uint32_t timeoutMs) 120 { 121 const auto currentMs = GetCurrentMillisecond(); 122 if (lastEventMillis_.load() <= 0) { 123 lastEventMillis_.store(currentMs); 124 } 125 if (currentMs - lastEventMillis_.load() >= idleThresholdMs) { 126 return true; 127 } 128 static constexpr auto sliceMs = 10; 129 this_thread::sleep_for(chrono::milliseconds(sliceMs)); 130 if (timeoutMs <= sliceMs) { 131 return false; 132 } 133 return WaitEventIdle(idleThresholdMs, timeoutMs - sliceMs); 134 } 135 SysUiController(string_view name,string_view device)136 SysUiController::SysUiController(string_view name, string_view device) : UiController(name, device) {} 137 ~SysUiController()138 SysUiController::~SysUiController() 139 { 140 DisConnectFromSysAbility(); 141 } 142 MarshalAccessibilityNodeAttributes(AccessibilityElementInfo & node,json & to)143 static void MarshalAccessibilityNodeAttributes(AccessibilityElementInfo &node, json &to) 144 { 145 to[ATTR_NAMES[UiAttr::TEXT]] = node.GetContent(); 146 to[ATTR_NAMES[UiAttr::ID]] = to_string(node.GetAccessibilityId()); 147 to[ATTR_NAMES[UiAttr::KEY]] = node.GetInspectorKey(); 148 to[ATTR_NAMES[UiAttr::TYPE]] = node.GetComponentType(); 149 to[ATTR_NAMES[UiAttr::ENABLED]] = node.IsEnabled() ? "true" : "false"; 150 to[ATTR_NAMES[UiAttr::FOCUSED]] = node.IsFocused() ? "true" : "false"; 151 to[ATTR_NAMES[UiAttr::SELECTED]] = node.IsSelected() ? "true" : "false"; 152 to[ATTR_NAMES[UiAttr::CLICKABLE]] = "false"; 153 to[ATTR_NAMES[UiAttr::LONG_CLICKABLE]] = "false"; 154 to[ATTR_NAMES[UiAttr::SCROLLABLE]] = "false"; 155 to["visible"] = node.IsVisible() ? "true" : "false"; 156 auto actionList = node.GetActionList(); 157 for (auto &action :actionList) { 158 switch (action.GetActionType()) { 159 case ACCESSIBILITY_ACTION_CLICK: 160 to[ATTR_NAMES[UiAttr::CLICKABLE]] = "true"; 161 break; 162 case ACCESSIBILITY_ACTION_LONG_CLICK: 163 to[ATTR_NAMES[UiAttr::LONG_CLICKABLE]] = "true"; 164 break; 165 case ACCESSIBILITY_ACTION_SCROLL_FORWARD: 166 case ACCESSIBILITY_ACTION_SCROLL_BACKWARD: 167 to[ATTR_NAMES[UiAttr::SCROLLABLE]] = "true"; 168 break; 169 default: 170 break; 171 } 172 } 173 } 174 MarshallAccessibilityNodeInfo(AccessibilityElementInfo & from,json & to)175 static void MarshallAccessibilityNodeInfo(AccessibilityElementInfo &from, json &to) 176 { 177 const auto rect = from.GetRectInScreen(); 178 const auto nodeBounds = Rect { 179 rect.GetLeftTopXScreenPostion(), rect.GetRightBottomXScreenPostion(), 180 rect.GetLeftTopYScreenPostion(), rect.GetRightBottomYScreenPostion()}; 181 json attributes; 182 MarshalAccessibilityNodeAttributes(from, attributes); 183 stringstream stream; 184 // "[%d,%d][%d,%d]", rect.left, rect.top, rect.right, rect.bottom 185 stream << "[" << nodeBounds.left_ << "," << nodeBounds.top_ << "]" 186 << "[" << nodeBounds.right_ << "," << nodeBounds.bottom_ << "]"; 187 attributes[ATTR_NAMES[UiAttr::BOUNDS]] = stream.str(); 188 to["attributes"] = attributes; 189 auto childList = json::array(); 190 const auto childCount = from.GetChildCount(); 191 AccessibilityElementInfo child; 192 for (auto index = 0; index < childCount; index++) { 193 auto success = from.GetChild(index, child); 194 if (success) { 195 if (!child.IsVisible()) { 196 continue; 197 } 198 auto parcel = json(); 199 MarshallAccessibilityNodeInfo(child, parcel); 200 childList.push_back(parcel); 201 } else { 202 LOG_W("Get Node child at index=%{public}d failed", index); 203 } 204 } 205 to["children"] = childList; 206 } 207 GetCurrentUiDom2(nlohmann::json & out)208 static void GetCurrentUiDom2(nlohmann::json& out) 209 { 210 auto ability = AccessibilityUITestAbility::GetInstance(); 211 std::optional<AccessibilityElementInfo> elementInfo; 212 ability->GetRootElementInfo(elementInfo); 213 if (elementInfo.has_value()) { 214 const auto windowId = elementInfo.value().GetWindowId(); 215 const auto windows = ability->GetWindows(); 216 for (auto& window:windows) { 217 if (windowId == window.GetWindowId()) { 218 // apply window bounds as root node bounds 219 auto windowRect = window.GetRectInScreen(); 220 elementInfo.value().SetRectInScreen(windowRect); 221 break; 222 } 223 } 224 MarshallAccessibilityNodeInfo(elementInfo.value(), out); 225 } else { 226 LOG_I("Root node not found"); 227 } 228 } 229 GetCurrentUiDom(nlohmann::json & out) const230 void SysUiController::GetCurrentUiDom(nlohmann::json& out) const 231 { 232 GetCurrentUiDom2(out); 233 } 234 WaitForUiSteady(uint32_t idleThresholdMs,uint32_t timeoutMs) const235 void SysUiController::WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutMs) const 236 { 237 } 238 InjectTouchEventSequence(const vector<TouchEvent> & events) const239 void SysUiController::InjectTouchEventSequence(const vector<TouchEvent> &events) const 240 { 241 for (auto& event:events) { 242 auto pointerEvent = PointerEvent::Create(); 243 PointerEvent::PointerItem pinterItem; 244 pinterItem.SetPointerId(0); 245 pinterItem.SetGlobalX(event.point_.px_); 246 pinterItem.SetGlobalY(event.point_.py_); 247 switch (event.stage_) { 248 case ActionStage::DOWN: 249 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_DOWN); 250 break; 251 case ActionStage::MOVE: 252 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_MOVE); 253 break; 254 case ActionStage::UP: 255 pointerEvent->SetPointerAction(PointerEvent::POINTER_ACTION_UP); 256 break; 257 } 258 pinterItem.SetPressed(event.stage_ != ActionStage::UP); 259 pointerEvent->AddPointerItem(pinterItem); 260 pointerEvent->SetSourceType(PointerEvent::SOURCE_TYPE_TOUCHSCREEN); 261 pointerEvent->SetPointerId(0); 262 InputManager::GetInstance()->SimulateInputEvent(pointerEvent); 263 if (event.holdMs_ > 0) { 264 this_thread::sleep_for(chrono::milliseconds(event.holdMs_)); 265 } 266 } 267 } 268 InjectKeyEventSequence(const vector<KeyEvent> & events) const269 void SysUiController::InjectKeyEventSequence(const vector<KeyEvent> &events) const 270 { 271 vector<int32_t> downKeys; 272 for (auto& event:events) { 273 if (event.stage_ == ActionStage::UP) { 274 auto iter = std::find(downKeys.begin(), downKeys.end(), event.code_); 275 if (iter == downKeys.end()) { 276 LOG_W("Cannot release a not-pressed key: %{public}d", event.code_); 277 continue; 278 } 279 downKeys.erase(iter); 280 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 281 keyEvent->SetKeyCode(event.code_); 282 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_UP); 283 OHOS::MMI::KeyEvent::KeyItem keyItem; 284 keyItem.SetKeyCode(event.code_); 285 keyItem.SetPressed(true); 286 keyEvent->AddKeyItem(keyItem); 287 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 288 } else { 289 downKeys.push_back(event.code_); 290 auto keyEvent = OHOS::MMI::KeyEvent::Create(); 291 for (auto downKey:downKeys) { 292 keyEvent->SetKeyCode(downKey); 293 keyEvent->SetKeyAction(OHOS::MMI::KeyEvent::KEY_ACTION_DOWN); 294 OHOS::MMI::KeyEvent::KeyItem keyItem; 295 keyItem.SetKeyCode(downKey); 296 keyItem.SetPressed(true); 297 keyEvent->AddKeyItem(keyItem); 298 } 299 InputManager::GetInstance()->SimulateInputEvent(keyEvent); 300 if (event.holdMs_ > 0) { 301 this_thread::sleep_for(chrono::milliseconds(event.holdMs_)); 302 } 303 } 304 } 305 // check not released keys 306 for (auto downKey:downKeys) { 307 LOG_W("Key event sequence injections done with not-released key: %{public}d", downKey); 308 } 309 } 310 PutTextToClipboard(string_view text) const311 void SysUiController::PutTextToClipboard(string_view text) const 312 { 313 } 314 IsWorkable() const315 bool SysUiController::IsWorkable() const 316 { 317 return connected_; 318 } 319 GetCharKeyCode(char ch,int32_t & code,int32_t & ctrlCode) const320 bool SysUiController::GetCharKeyCode(char ch, int32_t &code, int32_t &ctrlCode) const 321 { 322 static const map<char, int32_t> keyMap = { 323 {'*', OHOS::MMI::KeyEvent::KEYCODE_STAR}, 324 {'#', OHOS::MMI::KeyEvent::KEYCODE_POUND}, 325 {',', OHOS::MMI::KeyEvent::KEYCODE_COMMA}, 326 {'.', OHOS::MMI::KeyEvent::KEYCODE_PERIOD}, 327 {'`', OHOS::MMI::KeyEvent::KEYCODE_GRAVE}, 328 {'-', OHOS::MMI::KeyEvent::KEYCODE_MINUS}, 329 {'=', OHOS::MMI::KeyEvent::KEYCODE_EQUALS}, 330 {'[', OHOS::MMI::KeyEvent::KEYCODE_LEFT_BRACKET}, 331 {']', OHOS::MMI::KeyEvent::KEYCODE_RIGHT_BRACKET}, 332 {'\\', OHOS::MMI::KeyEvent::KEYCODE_BACKSLASH}, 333 {';', OHOS::MMI::KeyEvent::KEYCODE_SEMICOLON}, 334 {'\'', OHOS::MMI::KeyEvent::KEYCODE_APOSTROPHE}, 335 {'/', OHOS::MMI::KeyEvent::KEYCODE_SLASH}, 336 {'@', OHOS::MMI::KeyEvent::KEYCODE_AT}, 337 {'+', OHOS::MMI::KeyEvent::KEYCODE_PLUS}, 338 {' ', OHOS::MMI::KeyEvent::KEYCODE_TAB}, 339 {' ', OHOS::MMI::KeyEvent::KEYCODE_SPACE}, 340 {0x7F, OHOS::MMI::KeyEvent::KEYCODE_DEL}}; 341 ctrlCode = KEYCODE_NONE; 342 if (ch >= 'a' && ch <= 'z') { 343 code = OHOS::MMI::KeyEvent::KEYCODE_A + (ch - 'a'); 344 } else if (ch >= 'A' && ch <= 'Z') { 345 ctrlCode = OHOS::MMI::KeyEvent::KEYCODE_SHIFT_LEFT; 346 code = OHOS::MMI::KeyEvent::KEYCODE_A + (ch - 'A'); 347 } else if (ch >= '0' && ch <= '9') { 348 code = OHOS::MMI::KeyEvent::KEYCODE_0 + (ch - '0'); 349 } else { 350 auto find = keyMap.find(ch); 351 if (find != keyMap.end()) { 352 code = find->second; 353 } else { 354 LOG_W("No keyCode found for char '%{public}c'", ch); 355 return false; 356 } 357 } 358 return true; 359 } 360 TakeScreenCap(string_view savePath,stringstream & errReceiver) const361 bool SysUiController::TakeScreenCap(string_view savePath, stringstream &errReceiver) const 362 { 363 DisplayManager &displayMgr = DisplayManager::GetInstance(); 364 // get PixelMap from DisplayManager API 365 shared_ptr<PixelMap> pixelMap = displayMgr.GetScreenshot(displayMgr.GetDefaultDisplayId()); 366 if (pixelMap == nullptr) { 367 errReceiver << "Failed to get display pixelMap"; 368 return false; 369 } 370 FILE *fp = fopen(savePath.data(), "wb"); 371 if (fp == nullptr) { 372 perror("File opening failed"); 373 errReceiver << "File opening failed: " << savePath; 374 return false; 375 } 376 png_structp pngStruct = png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); 377 if (pngStruct == nullptr) { 378 fclose(fp); 379 return false; 380 } 381 png_infop pngInfo = png_create_info_struct(pngStruct); 382 if (pngInfo == nullptr) { 383 fclose(fp); 384 png_destroy_write_struct(&pngStruct, nullptr); 385 return false; 386 } 387 png_init_io(pngStruct, fp); 388 auto width = static_cast<uint32_t>(pixelMap->GetWidth()); 389 auto height = static_cast<uint32_t>(pixelMap->GetHeight()); 390 auto data = pixelMap->GetPixels(); 391 auto stride = static_cast<uint32_t>(pixelMap->GetRowBytes()); 392 // set png header 393 static constexpr int bitmapDepth = 8; 394 png_set_IHDR(pngStruct, pngInfo, width, height, bitmapDepth, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, 395 PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE); 396 png_set_packing(pngStruct); // set packing info 397 png_write_info(pngStruct, pngInfo); // write to header 398 for (uint32_t column = 0; column < height; column++) { 399 png_write_row(pngStruct, data + (column * stride)); 400 } 401 // free/close 402 png_write_end(pngStruct, pngInfo); 403 png_destroy_write_struct(&pngStruct, &pngInfo); 404 (void)fclose(fp); 405 return true; 406 } 407 408 // UiEventMonitor instance. 409 static shared_ptr<UiEventMonitor> g_monitorInstance_; 410 ConnectToSysAbility()411 bool SysUiController::ConnectToSysAbility() 412 { 413 if (connected_) { 414 return true; 415 } 416 mutex mtx; 417 unique_lock<mutex> uLock(mtx); 418 condition_variable condition; 419 auto onConnectCallback = [&condition]() { 420 LOG_I("Success connect to AAMS"); 421 condition.notify_all(); 422 }; 423 if (g_monitorInstance_ == nullptr) { 424 g_monitorInstance_ = make_shared<UiEventMonitor>(); 425 } 426 g_monitorInstance_->SetOnAbilityConnectCallback(onConnectCallback); 427 auto ability = AccessibilityUITestAbility::GetInstance(); 428 if (!ability->RegisterListener(g_monitorInstance_)) { 429 LOG_E("Failed to register UiEventMonitor"); 430 return false; 431 } 432 LOG_I("Start connect to AAMS"); 433 if (!ability->Connect()) { 434 LOG_E("Failed to connect to AAMS"); 435 return false; 436 } 437 const auto timeout = chrono::milliseconds(500); 438 if (condition.wait_for(uLock, timeout) == cv_status::timeout) { 439 LOG_E("Wait connection to AAMS timed out"); 440 return false; 441 } 442 LOG_I("Register to AAMS done"); 443 connected_ = true; 444 return true; 445 } 446 DisConnectFromSysAbility()447 void SysUiController::DisConnectFromSysAbility() 448 { 449 if (!connected_ || g_monitorInstance_ == nullptr) { 450 return; 451 } 452 connected_ = false; 453 mutex mtx; 454 unique_lock<mutex> uLock(mtx); 455 condition_variable condition; 456 auto onDisConnectCallback = [&condition]() { 457 LOG_I("Success disconnect from AAMS"); 458 condition.notify_all(); 459 }; 460 g_monitorInstance_->SetOnAbilityDisConnectCallback(onDisConnectCallback); 461 auto ability = AccessibilityUITestAbility::GetInstance(); 462 LOG_I("Start disconnect from AAMS"); 463 if (! ability->Disconnect()) { 464 LOG_E("Failed to disconnect from AAMS"); 465 return; 466 } 467 const auto timeout = chrono::milliseconds(200); 468 if (condition.wait_for(uLock, timeout) == cv_status::timeout) { 469 LOG_E("Wait disconnection from AAMS timed out"); 470 return; 471 } 472 } 473 }