1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #include <future> 17 #include "ui_model.h" 18 #include "ui_driver.h" 19 20 namespace OHOS::uitest { 21 using namespace std; 22 using namespace nlohmann; 23 24 class WindowCacheCompareGreater { 25 public: operator ()(const WindowCacheModel & w1,const WindowCacheModel & w2)26 bool operator()(const WindowCacheModel &w1, const WindowCacheModel &w2) 27 { 28 if (w1.window_.actived_) { 29 return true; 30 } 31 if (w2.window_.actived_) { 32 return false; 33 } 34 if (w1.window_.focused_) { 35 return true; 36 } 37 if (w2.window_.focused_) { 38 return false; 39 } 40 return w1.window_.windowLayer_ > w2.window_.windowLayer_; 41 } 42 }; 43 44 std::unique_ptr<UiController> UiDriver::uiController_; 45 RegisterController(std::unique_ptr<UiController> controller)46 void UiDriver::RegisterController(std::unique_ptr<UiController> controller) 47 { 48 uiController_ = move(controller); 49 } 50 RegisterUiEventListener(std::shared_ptr<UiEventListener> listener)51 void UiDriver::RegisterUiEventListener(std::shared_ptr<UiEventListener> listener) 52 { 53 uiController_->RegisterUiEventListener(listener); 54 } 55 CheckStatus(bool isConnected,ApiCallErr & error)56 bool UiDriver::CheckStatus(bool isConnected, ApiCallErr &error) 57 { 58 DCHECK(uiController_); 59 if (isConnected && !uiController_->IsWorkable()) { 60 LOG_W("Not connect to AAMS, try to reconnect"); 61 if (!uiController_->Initialize()) { 62 error = ApiCallErr(ERR_INITIALIZE_FAILED, "Can not connect to AAMS"); 63 return false; 64 } 65 } 66 return true; 67 } 68 UpdateUIWindows(ApiCallErr & error)69 void UiDriver::UpdateUIWindows(ApiCallErr &error) 70 { 71 visitWidgets_.clear(); 72 targetWidgetsIndex_.clear(); 73 windowCacheVec_.clear(); 74 if (!CheckStatus(true, error)) { 75 return; 76 } 77 std::vector<Window> currentWindowVec; 78 uiController_->GetUiWindows(currentWindowVec); 79 if (currentWindowVec.empty()) { 80 LOG_E("Get Windows Failed"); 81 error = ApiCallErr(ERR_INTERNAL, "Get window nodes failed"); 82 } 83 84 for (const auto &win : currentWindowVec) { 85 WindowCacheModel cacheModel(win); 86 windowCacheVec_.emplace_back(std::move(cacheModel)); 87 std::stringstream ss; 88 ss << "window rect is "; 89 ss << win.bounds_.Describe(); 90 ss << "overplay window rects are:["; 91 for (const auto& overRect : win.invisibleBoundsVec_) { 92 ss << overRect.Describe(); 93 ss << ", "; 94 } 95 ss << "]"; 96 LOG_D("window id is %{public}d, rect info is %{public}s", win.id_, ss.str().data()); 97 } 98 // actice or focus window move to top 99 std::sort(windowCacheVec_.begin(), windowCacheVec_.end(), WindowCacheCompareGreater()); 100 } 101 DumpWindowsInfo(bool listWindows,Rect & mergeBounds,nlohmann::json & childDom)102 void UiDriver::DumpWindowsInfo(bool listWindows, Rect& mergeBounds, nlohmann::json& childDom) 103 { 104 std::vector<WidgetMatchModel> emptyMatcher; 105 StrategyBuildParam buildParam; 106 buildParam.myselfMatcher = emptyMatcher; 107 std::unique_ptr<SelectStrategy> selectStrategy = SelectStrategy::BuildSelectStrategy(buildParam, true); 108 for (auto &winCache : windowCacheVec_) { 109 visitWidgets_.clear(); 110 targetWidgetsIndex_.clear(); 111 if (!uiController_->GetWidgetsInWindow(winCache.window_, winCache.widgetIterator_)) { 112 LOG_W("Get Widget from window[%{public}d] failed, skip the window", winCache.window_.id_); 113 continue; 114 } 115 selectStrategy->LocateNode(winCache.window_, *winCache.widgetIterator_, visitWidgets_, targetWidgetsIndex_, 116 !listWindows); 117 nlohmann::json child = nlohmann::json(); 118 if (visitWidgets_.empty()) { 119 LOG_E("Window %{public}s has no node, skip it", winCache.window_.bundleName_.data()); 120 continue; 121 } else { 122 DumpHandler::DumpWindowInfoToJson(visitWidgets_, child); 123 } 124 child["attributes"]["abilityName"] = winCache.window_.abilityName_; 125 child["attributes"]["bundleName"] = winCache.window_.bundleName_; 126 child["attributes"]["pagePath"] = winCache.window_.pagePath_; 127 childDom.emplace_back(child); 128 mergeBounds.left_ = std::min(mergeBounds.left_, winCache.window_.bounds_.left_); 129 mergeBounds.top_ = std::min(mergeBounds.top_, winCache.window_.bounds_.top_); 130 mergeBounds.right_ = std::max(mergeBounds.right_, winCache.window_.bounds_.right_); 131 mergeBounds.bottom_ = std::max(mergeBounds.bottom_, winCache.window_.bounds_.bottom_); 132 } 133 } 134 DumpUiHierarchy(nlohmann::json & out,bool listWindows,bool addExternAttr,ApiCallErr & error)135 void UiDriver::DumpUiHierarchy(nlohmann::json &out, bool listWindows, bool addExternAttr, ApiCallErr &error) 136 { 137 UpdateUIWindows(error); 138 if (error.code_ != NO_ERROR) { 139 return; 140 } 141 nlohmann::json childDom = nlohmann::json::array(); 142 Rect mergeBounds{0, 0, 0, 0}; 143 DumpWindowsInfo(listWindows, mergeBounds, childDom); 144 if (listWindows) { 145 out = childDom; 146 } else { 147 nlohmann::json attrData = nlohmann::json(); 148 149 for (int i = 0; i < UiAttr::HIERARCHY; ++i) { 150 attrData[ATTR_NAMES[i].data()] = ""; 151 } 152 std::stringstream ss; 153 ss << "[" << mergeBounds.left_ << "," << mergeBounds.top_ << "]" 154 << "[" << mergeBounds.right_ << "," << mergeBounds.bottom_ << "]"; 155 attrData[ATTR_NAMES[UiAttr::BOUNDS].data()] = ss.str(); 156 out["attributes"] = attrData; 157 out["children"] = childDom; 158 } 159 160 if (addExternAttr) { 161 map<int32_t, string_view> elementTrees; 162 vector<char *> buffers; 163 for (auto &winCache : windowCacheVec_) { 164 char *buffer = nullptr; 165 size_t len = 0; 166 uiController_->GetHidumperInfo(to_string(winCache.window_.id_), &buffer, len); 167 if (buffer == nullptr) { 168 continue; 169 } 170 elementTrees.insert(make_pair(winCache.window_.id_, string_view(buffer, len))); 171 buffers.push_back(buffer); 172 } 173 DumpHandler::AddExtraAttrs(out, elementTrees, 0); 174 for (auto &buf : buffers) { 175 delete buf; 176 } 177 } 178 } 179 CloneFreeWidget(const Widget & from,const string & selectDesc)180 static unique_ptr<Widget> CloneFreeWidget(const Widget &from, const string &selectDesc) 181 { 182 auto clone = from.Clone(from.GetHierarchy()); 183 clone->SetAttr(UiAttr::DUMMY_ATTRNAME_SELECTION, selectDesc + from.GetAttr(UiAttr::HASHCODE)); 184 // save the selection desc as dummy attribute 185 return clone; 186 } 187 ConstructSelectStrategyByRetrieve(const Widget & widget)188 static std::unique_ptr<SelectStrategy> ConstructSelectStrategyByRetrieve(const Widget &widget) 189 { 190 WidgetMatchModel attrMatch{UiAttr::HASHCODE, widget.GetAttr(UiAttr::HASHCODE), EQ}; 191 StrategyBuildParam buildParam; 192 buildParam.myselfMatcher.emplace_back(attrMatch); 193 return SelectStrategy::BuildSelectStrategy(buildParam, false); 194 } 195 GetHostApp(const Widget & widget)196 string UiDriver::GetHostApp(const Widget &widget) 197 { 198 auto winId = widget.GetAttr(UiAttr::HOST_WINDOW_ID); 199 if (winId.length() < 1) { 200 winId = "0"; 201 } 202 auto id = atoi(winId.c_str()); 203 for (auto &windowCache : windowCacheVec_) { 204 if (id == windowCache.window_.id_) { 205 // If not a actived window, get all. 206 if (windowCache.window_.actived_ == false) { 207 return ""; 208 } 209 return windowCache.window_.bundleName_; 210 } 211 } 212 return ""; 213 } 214 RetrieveWidget(const Widget & widget,ApiCallErr & err,bool updateUi)215 const Widget *UiDriver::RetrieveWidget(const Widget &widget, ApiCallErr &err, bool updateUi) 216 { 217 if (updateUi) { 218 UpdateUIWindows(err); 219 if (err.code_ != NO_ERROR) { 220 LOG_I("Retrieve Widget with error %{public}s", err.message_.c_str()); 221 return nullptr; 222 } 223 } else { 224 visitWidgets_.clear(); 225 targetWidgetsIndex_.clear(); 226 } 227 228 std::unique_ptr<SelectStrategy> selectStrategy = ConstructSelectStrategyByRetrieve(widget); 229 for (auto &curWinCache : windowCacheVec_) { 230 if (widget.GetAttr(UiAttr::HOST_WINDOW_ID) != std::to_string(curWinCache.window_.id_)) { 231 continue; 232 } 233 selectStrategy->SetAndCalcSelectWindowRect(curWinCache.window_.bounds_, 234 curWinCache.window_.invisibleBoundsVec_); 235 if (curWinCache.widgetIterator_ == nullptr) { 236 if (!uiController_->GetWidgetsInWindow(curWinCache.window_, curWinCache.widgetIterator_)) { 237 LOG_W("Get Widget from window[%{public}d] failed, skip the window", curWinCache.window_.id_); 238 continue; 239 } 240 } 241 selectStrategy->LocateNode(curWinCache.window_, *curWinCache.widgetIterator_, visitWidgets_, 242 targetWidgetsIndex_); 243 if (!targetWidgetsIndex_.empty()) { 244 break; 245 } 246 } 247 stringstream msg; 248 msg << "Widget: " << widget.GetAttr(UiAttr::DUMMY_ATTRNAME_SELECTION); 249 msg << "dose not exist on current UI! Check if the UI has changed after you got the widget object"; 250 if (targetWidgetsIndex_.empty()) { 251 msg << "(NoCandidates)"; 252 err = ApiCallErr(ERR_COMPONENT_LOST, msg.str()); 253 LOG_W("%{public}s", err.message_.c_str()); 254 return nullptr; 255 } 256 DCHECK(targetWidgetsIndex_.size() == 1); 257 // confirm type 258 if (widget.GetAttr(UiAttr::TYPE) != visitWidgets_[targetWidgetsIndex_[0]].GetAttr(UiAttr::TYPE)) { 259 msg << " (CompareEqualsFailed)"; 260 err = ApiCallErr(ERR_COMPONENT_LOST, msg.str()); 261 LOG_W("%{public}s", err.message_.c_str()); 262 return nullptr; 263 } 264 return &visitWidgets_[targetWidgetsIndex_[0]]; 265 } 266 TriggerKey(const KeyAction & key,const UiOpArgs & opt,ApiCallErr & error)267 void UiDriver::TriggerKey(const KeyAction &key, const UiOpArgs &opt, ApiCallErr &error) 268 { 269 if (!CheckStatus(false, error)) { 270 return; 271 } 272 vector<KeyEvent> events; 273 key.ComputeEvents(events, opt); 274 if (events.empty()) { 275 return; 276 } 277 uiController_->InjectKeyEventSequence(events); 278 uiController_->WaitForUiSteady(opt.uiSteadyThresholdMs_, opt.waitUiSteadyMaxMs_); 279 } 280 FindWidgets(const WidgetSelector & selector,vector<unique_ptr<Widget>> & rev,ApiCallErr & err,bool updateUi)281 void UiDriver::FindWidgets(const WidgetSelector &selector, vector<unique_ptr<Widget>> &rev, 282 ApiCallErr &err, bool updateUi) 283 { 284 if (updateUi) { 285 UpdateUIWindows(err); 286 if (err.code_ != NO_ERROR) { 287 return; 288 } 289 } else { 290 visitWidgets_.clear(); 291 targetWidgetsIndex_.clear(); 292 } 293 for (auto &curWinCache : windowCacheVec_) { 294 LOG_I("Start find in Window, window id is %{public}d", curWinCache.window_.id_); 295 if (curWinCache.widgetIterator_ == nullptr) { 296 std::unique_ptr<ElementNodeIterator> widgetIterator = nullptr; 297 if (!uiController_->GetWidgetsInWindow(curWinCache.window_, curWinCache.widgetIterator_)) { 298 LOG_W("Get Widget from window[%{public}d] failed, skip the window", curWinCache.window_.id_); 299 continue; 300 } 301 } 302 selector.Select(curWinCache.window_, *curWinCache.widgetIterator_, visitWidgets_, targetWidgetsIndex_); 303 if (!selector.IsWantMulti() && !targetWidgetsIndex_.empty()) { 304 break; 305 } 306 if (!selector.IsWantMulti()) { 307 visitWidgets_.clear(); 308 targetWidgetsIndex_.clear(); 309 } 310 } 311 if (targetWidgetsIndex_.empty()) { 312 LOG_W("self node not found by %{public}s", selector.Describe().data()); 313 return; 314 } 315 // covert widgets to images as return value 316 uint32_t index = 0; 317 for (auto targetIndex : targetWidgetsIndex_) { 318 auto image = CloneFreeWidget(visitWidgets_[targetIndex], selector.Describe()); 319 // at sometime, more than one widgets are found, add the node index to the description 320 rev.emplace_back(move(image)); 321 index++; 322 } 323 } 324 WaitForWidget(const WidgetSelector & selector,const UiOpArgs & opt,ApiCallErr & err)325 unique_ptr<Widget> UiDriver::WaitForWidget(const WidgetSelector &selector, const UiOpArgs &opt, ApiCallErr &err) 326 { 327 const uint32_t sliceMs = 20; 328 const auto startMs = GetCurrentMillisecond(); 329 vector<unique_ptr<Widget>> receiver; 330 do { 331 FindWidgets(selector, receiver, err); 332 if (err.code_ != NO_ERROR) { // abort on error 333 return nullptr; 334 } 335 if (!receiver.empty()) { 336 return move(receiver.at(0)); 337 } 338 DelayMs(sliceMs); 339 } while (GetCurrentMillisecond() - startMs < opt.waitWidgetMaxMs_); 340 return nullptr; 341 } 342 DelayMs(uint32_t ms)343 void UiDriver::DelayMs(uint32_t ms) 344 { 345 if (ms > 0) { 346 this_thread::sleep_for(chrono::milliseconds(ms)); 347 } 348 } 349 PerformTouch(const TouchAction & touch,const UiOpArgs & opt,ApiCallErr & err)350 void UiDriver::PerformTouch(const TouchAction &touch, const UiOpArgs &opt, ApiCallErr &err) 351 { 352 if (!CheckStatus(false, err)) { 353 return; 354 } 355 PointerMatrix events; 356 touch.Decompose(events, opt); 357 if (events.Empty()) { 358 return; 359 } 360 uiController_->InjectTouchEventSequence(events); 361 uiController_->WaitForUiSteady(opt.uiSteadyThresholdMs_, opt.waitUiSteadyMaxMs_); 362 } 363 PerformMouseAction(const MouseAction & touch,const UiOpArgs & opt,ApiCallErr & err)364 void UiDriver::PerformMouseAction(const MouseAction &touch, const UiOpArgs &opt, ApiCallErr &err) 365 { 366 if (!CheckStatus(false, err)) { 367 return; 368 } 369 vector<MouseEvent> events; 370 touch.Decompose(events, opt); 371 if (events.empty()) { 372 return; 373 } 374 uiController_->InjectMouseEventSequence(events); 375 uiController_->WaitForUiSteady(opt.uiSteadyThresholdMs_, opt.waitUiSteadyMaxMs_); 376 } 377 TakeScreenCap(int32_t fd,ApiCallErr & err,Rect rect)378 void UiDriver::TakeScreenCap(int32_t fd, ApiCallErr &err, Rect rect) 379 { 380 if (!CheckStatus(false, err)) { 381 return; 382 } 383 stringstream errorRecv; 384 if (!uiController_->TakeScreenCap(fd, errorRecv, rect)) { 385 string errStr = errorRecv.str(); 386 LOG_W("ScreenCap failed: %{public}s", errStr.c_str()); 387 if (errStr.find("File opening failed") == 0) { 388 err = ApiCallErr(ERR_INVALID_INPUT, "Invalid save path or permission denied"); 389 } else { 390 err = ApiCallErr(ERR_INTERNAL, errStr); 391 } 392 LOG_W("ScreenCap failed: %{public}s", errorRecv.str().c_str()); 393 } else { 394 LOG_D("ScreenCap successed"); 395 } 396 } 397 FindWindow(function<bool (const Window &)> matcher,ApiCallErr & err)398 unique_ptr<Window> UiDriver::FindWindow(function<bool(const Window &)> matcher, ApiCallErr &err) 399 { 400 UpdateUIWindows(err); 401 if (err.code_ != NO_ERROR) { 402 return nullptr; 403 } 404 for (auto &windowCache : windowCacheVec_) { 405 if (matcher(windowCache.window_)) { 406 auto clone = make_unique<Window>(0); 407 *clone = windowCache.window_; // copy construct 408 return clone; 409 } 410 } 411 return nullptr; 412 } 413 RetrieveWindow(const Window & window,ApiCallErr & err)414 const Window *UiDriver::RetrieveWindow(const Window &window, ApiCallErr &err) 415 { 416 UpdateUIWindows(err); 417 if (err.code_ != NO_ERROR) { 418 return nullptr; 419 } 420 for (auto &winCache : windowCacheVec_) { 421 if (winCache.window_.id_ != window.id_) { 422 continue; 423 } 424 return &winCache.window_; 425 } 426 stringstream msg; 427 msg << "Window " << window.id_; 428 msg << "dose not exist on current UI! Check if the UI has changed after you got the window object"; 429 err = ApiCallErr(ERR_COMPONENT_LOST, msg.str()); 430 LOG_W("%{public}s", err.message_.c_str()); 431 return nullptr; 432 } 433 SetDisplayRotation(DisplayRotation rotation,ApiCallErr & error)434 void UiDriver::SetDisplayRotation(DisplayRotation rotation, ApiCallErr &error) 435 { 436 if (!CheckStatus(false, error)) { 437 return; 438 } 439 uiController_->SetDisplayRotation(rotation); 440 } 441 GetDisplayRotation(ApiCallErr & error)442 DisplayRotation UiDriver::GetDisplayRotation(ApiCallErr &error) 443 { 444 if (!CheckStatus(false, error)) { 445 return ROTATION_0; 446 } 447 return uiController_->GetDisplayRotation(); 448 } 449 SetDisplayRotationEnabled(bool enabled,ApiCallErr & error)450 void UiDriver::SetDisplayRotationEnabled(bool enabled, ApiCallErr &error) 451 { 452 if (!CheckStatus(false, error)) { 453 return; 454 } 455 uiController_->SetDisplayRotationEnabled(enabled); 456 } 457 WaitForUiSteady(uint32_t idleThresholdMs,uint32_t timeoutSec,ApiCallErr & error)458 bool UiDriver::WaitForUiSteady(uint32_t idleThresholdMs, uint32_t timeoutSec, ApiCallErr &error) 459 { 460 if (!CheckStatus(false, error)) { 461 return false; 462 } 463 return uiController_->WaitForUiSteady(idleThresholdMs, timeoutSec); 464 } 465 WakeUpDisplay(ApiCallErr & error)466 void UiDriver::WakeUpDisplay(ApiCallErr &error) 467 { 468 if (!CheckStatus(false, error)) { 469 return; 470 } 471 if (uiController_->IsScreenOn()) { 472 return; 473 } else { 474 LOG_I("screen is off, turn it on"); 475 UiOpArgs uiOpArgs; 476 this->TriggerKey(Power(), uiOpArgs, error); 477 } 478 } 479 GetDisplaySize(ApiCallErr & error)480 Point UiDriver::GetDisplaySize(ApiCallErr &error) 481 { 482 if (!CheckStatus(false, error)) { 483 return Point(0, 0); 484 } 485 return uiController_->GetDisplaySize(); 486 } 487 GetDisplayDensity(ApiCallErr & error)488 Point UiDriver::GetDisplayDensity(ApiCallErr &error) 489 { 490 if (!CheckStatus(false, error)) { 491 return Point(0, 0); 492 } 493 return uiController_->GetDisplayDensity(); 494 } 495 TextToKeyEvents(string_view text,std::vector<KeyEvent> & events,ApiCallErr & error)496 bool UiDriver::TextToKeyEvents(string_view text, std::vector<KeyEvent> &events, ApiCallErr &error) 497 { 498 if (!CheckStatus(false, error)) { 499 return false; 500 } 501 static constexpr uint32_t typeCharTimeMs = 50; 502 if (!text.empty()) { 503 vector<char> chars(text.begin(), text.end()); // decompose to sing-char input sequence 504 vector<pair<int32_t, int32_t>> keyCodes; 505 for (auto ch : chars) { 506 int32_t code = KEYCODE_NONE; 507 int32_t ctrlCode = KEYCODE_NONE; 508 if (!uiController_->GetCharKeyCode(ch, code, ctrlCode)) { 509 return false; 510 } 511 keyCodes.emplace_back(make_pair(code, ctrlCode)); 512 } 513 for (auto &pair : keyCodes) { 514 if (pair.second != KEYCODE_NONE) { 515 events.emplace_back(KeyEvent {ActionStage::DOWN, pair.second, 0}); 516 } 517 events.emplace_back(KeyEvent {ActionStage::DOWN, pair.first, typeCharTimeMs}); 518 events.emplace_back(KeyEvent {ActionStage::UP, pair.first, 0}); 519 if (pair.second != KEYCODE_NONE) { 520 events.emplace_back(KeyEvent {ActionStage::UP, pair.second, 0}); 521 } 522 } 523 } 524 return true; 525 } 526 InputText(string_view text,ApiCallErr & error)527 void UiDriver::InputText(string_view text, ApiCallErr &error) 528 { 529 vector<KeyEvent> events; 530 UiOpArgs uiOpArgs; 531 if (!text.empty()) { 532 if (TextToKeyEvents(text, events, error)) { 533 LOG_I("inputText by Keycode"); 534 auto keyActionForInput = KeysForwarder(events); 535 TriggerKey(keyActionForInput, uiOpArgs, error); 536 } else { 537 uiController_->PutTextToClipboard(text); 538 LOG_I("inputText by pasteBoard"); 539 auto actionForPatse = CombinedKeys(KEYCODE_CTRL, KEYCODE_V, KEYCODE_NONE); 540 TriggerKey(actionForPatse, uiOpArgs, error); 541 } 542 } 543 } 544 GetMergeWindowBounds(Rect & mergeRect)545 void UiDriver::GetMergeWindowBounds(Rect &mergeRect) 546 { 547 for (const auto &winCache : windowCacheVec_) { 548 mergeRect.left_ = std::min(winCache.window_.bounds_.left_, mergeRect.left_); 549 mergeRect.top_ = std::min(winCache.window_.bounds_.top_, mergeRect.top_); 550 mergeRect.right_ = std::max(winCache.window_.bounds_.right_, mergeRect.right_); 551 mergeRect.bottom_ = std::max(winCache.window_.bounds_.bottom_, mergeRect.bottom_); 552 } 553 } 554 } // namespace OHOS::uitest 555