1 /* 2 * Copyright (C) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef ACCESSIBILITY_TOUCH_EXPLORATION_H 17 #define ACCESSIBILITY_TOUCH_EXPLORATION_H 18 19 #include <string> 20 #include <cmath> 21 #include <vector> 22 #include <map> 23 #include <functional> 24 #include "accessibility_element_info.h" 25 #include "accessibility_element_operator_callback_stub.h" 26 #include "accessibility_event_transmission.h" 27 #include "accessible_ability_manager_service.h" 28 #include "accessibility_def.h" 29 #include "event_handler.h" 30 #include "hilog_wrapper.h" 31 32 namespace OHOS { 33 namespace Accessibility { 34 class TouchExploration; 35 36 const uint32_t MAX_MULTI_FINGER_TYPE = 3; 37 const int32_t LIMIT_SIZE_TWO = 2; 38 const int32_t LIMIT_SIZE_THREE = 3; 39 const int32_t DIRECTION_NUM = 4; 40 const int32_t TAP_COUNT_MAXIMUM = 3; 41 const int32_t MINI_POINTER_DISTANCE_DIP = 200; 42 const float DEGREES_THRESHOLD = 0.0f; 43 const uint32_t MIN_MULTI_FINGER_SWIPE_POINTER_NUM = 2; 44 const float TOUCH_SLOP = 8.0f; 45 const float MULTI_TAP_SLOP = 100.0f; 46 const float MULTI_TAP_SLOP_DELTA = 0.5f; 47 const int32_t SCREEN_AXIS_NUM = 2; 48 const double MAX_DRAG_GESTURE_COSINE = 0.525321989; 49 const double EPSINON = 0.01; 50 const float PIXEL_MULTIPLIER = 0.1f; 51 const int32_t DIVIDE_NUM = 2; 52 const uint32_t FIND_FOCUS_TIMEOUT = 50; 53 const int32_t SIMULATE_POINTER_ID = 10000; 54 #define BIND(func) [this](MMI::PointerEvent& event) { (func(event)); } 55 56 /** 57 * @brief touch exploration state define 58 */ 59 enum class TouchExplorationState : int32_t { 60 TOUCH_INIT, 61 PASSING_THROUGH, 62 INVALID, 63 64 ONE_FINGER_DOWN, 65 ONE_FINGER_LONG_PRESS, 66 ONE_FINGER_SWIPE, 67 ONE_FINGER_SINGLE_TAP, 68 ONE_FINGER_SINGLE_TAP_THEN_DOWN, 69 ONE_FINGER_DOUBLE_TAP_AND_LONG_PRESS, 70 71 TWO_FINGERS_DOWN, 72 TWO_FINGERS_DRAG, 73 TWO_FINGERS_TAP, 74 TWO_FINGERS_CONTINUE_DOWN, 75 TWO_FINGERS_UNKNOWN, 76 77 THREE_FINGERS_DOWN, 78 THREE_FINGERS_SWIPE, 79 THREE_FINGERS_TAP, 80 THREE_FINGERS_CONTINUE_DOWN, 81 82 FOUR_FINGERS_DOWN, 83 FOUR_FINGERS_SWIPE, 84 FOUR_FINGERS_TAP, 85 FOUR_FINGERS_CONTINUE_DOWN 86 }; 87 88 enum class ChangeAction : int32_t { 89 NO_CHANGE, 90 HOVER_MOVE, 91 POINTER_DOWN, 92 POINTER_UP, 93 HOVER_ENTER, 94 HOVER_EXIT, 95 HOVER_CANCEL 96 }; 97 98 enum class PointerCount : uint32_t { 99 POINTER_COUNT_1 = 1, 100 POINTER_COUNT_2 = 2, 101 POINTER_COUNT_3 = 3, 102 POINTER_COUNT_4 = 4 103 }; 104 105 enum class TimeoutDuration : int64_t { 106 LONG_PRESS_TIMEOUT = 200, 107 DOUBLE_TAP_TIMEOUT = 300, 108 MULTI_FINGER_TAP_INTERVAL_TIMEOUT = 100, 109 SWIPE_COMPLETE_TIMEOUT = 300 110 }; 111 112 enum class TouchExplorationMsg : uint32_t { 113 SEND_HOVER_MSG, 114 LONG_PRESS_MSG, 115 DOUBLE_TAP_AND_LONG_PRESS_MSG, 116 SWIPE_COMPLETE_TIMEOUT_MSG, 117 TWO_FINGER_SINGLE_TAP_MSG, 118 TWO_FINGER_LONG_PRESS_MSG, 119 TWO_FINGER_DOUBLE_TAP_MSG, 120 TWO_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 121 TWO_FINGER_TRIPLE_TAP_MSG, 122 TWO_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 123 THREE_FINGER_SINGLE_TAP_MSG, 124 THREE_FINGER_LONG_PRESS_MSG, 125 THREE_FINGER_DOUBLE_TAP_MSG, 126 THREE_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 127 THREE_FINGER_TRIPLE_TAP_MSG, 128 THREE_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 129 FOUR_FINGER_SINGLE_TAP_MSG, 130 FOUR_FINGER_LONG_PRESS_MSG, 131 FOUR_FINGER_DOUBLE_TAP_MSG, 132 FOUR_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 133 FOUR_FINGER_TRIPLE_TAP_MSG, 134 FOUR_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 135 WAIT_ANOTHER_FINGER_DOWN_MSG 136 }; 137 138 struct Pointer { 139 float px_; 140 float py_; 141 }; 142 143 class TouchExplorationEventHandler : public AppExecFwk::EventHandler { 144 public: 145 TouchExplorationEventHandler(const std::shared_ptr<AppExecFwk::EventRunner> &runner, 146 TouchExploration &tgServer); 147 virtual ~TouchExplorationEventHandler() = default; 148 /** 149 * @brief Process the event of install system bundles. 150 * @param event Indicates the event to be processed. 151 */ 152 virtual void ProcessEvent(const AppExecFwk::InnerEvent::Pointer &event) override; 153 154 private: 155 TouchExploration &server_; 156 }; 157 158 class TouchExploration : public EventTransmission { 159 public: 160 static constexpr GestureType GESTURE_DIRECTION[DIRECTION_NUM] = { 161 GestureType::GESTURE_SWIPE_UP, 162 GestureType::GESTURE_SWIPE_DOWN, 163 GestureType::GESTURE_SWIPE_LEFT, 164 GestureType::GESTURE_SWIPE_RIGHT 165 }; 166 167 static constexpr GestureType GESTURE_DIRECTION_TO_ID[DIRECTION_NUM][DIRECTION_NUM] = { 168 { 169 GestureType::GESTURE_SWIPE_UP, 170 GestureType::GESTURE_SWIPE_UP_THEN_DOWN, 171 GestureType::GESTURE_SWIPE_UP_THEN_LEFT, 172 GestureType::GESTURE_SWIPE_UP_THEN_RIGHT, 173 }, 174 { 175 GestureType::GESTURE_SWIPE_DOWN_THEN_UP, 176 GestureType::GESTURE_SWIPE_DOWN, 177 GestureType::GESTURE_SWIPE_DOWN_THEN_LEFT, 178 GestureType::GESTURE_SWIPE_DOWN_THEN_RIGHT, 179 180 }, 181 { 182 GestureType::GESTURE_SWIPE_LEFT_THEN_UP, 183 GestureType::GESTURE_SWIPE_LEFT_THEN_DOWN, 184 GestureType::GESTURE_SWIPE_LEFT, 185 GestureType::GESTURE_SWIPE_LEFT_THEN_RIGHT, 186 187 }, 188 { 189 GestureType::GESTURE_SWIPE_RIGHT_THEN_UP, 190 GestureType::GESTURE_SWIPE_RIGHT_THEN_DOWN, 191 GestureType::GESTURE_SWIPE_RIGHT_THEN_LEFT, 192 GestureType::GESTURE_SWIPE_RIGHT 193 } 194 }; 195 196 static constexpr TouchExplorationMsg GESTURE_TAP_MSG[TAP_COUNT_MAXIMUM][MAX_MULTI_FINGER_TYPE] = { 197 { 198 TouchExplorationMsg::TWO_FINGER_SINGLE_TAP_MSG, 199 TouchExplorationMsg::THREE_FINGER_SINGLE_TAP_MSG, 200 TouchExplorationMsg::FOUR_FINGER_SINGLE_TAP_MSG, 201 }, 202 { 203 TouchExplorationMsg::TWO_FINGER_DOUBLE_TAP_MSG, 204 TouchExplorationMsg::THREE_FINGER_DOUBLE_TAP_MSG, 205 TouchExplorationMsg::FOUR_FINGER_DOUBLE_TAP_MSG, 206 }, 207 { 208 TouchExplorationMsg::TWO_FINGER_TRIPLE_TAP_MSG, 209 TouchExplorationMsg::THREE_FINGER_TRIPLE_TAP_MSG, 210 TouchExplorationMsg::FOUR_FINGER_TRIPLE_TAP_MSG, 211 } 212 }; 213 214 static constexpr TouchExplorationMsg GESTURE_HOLD_MSG[TAP_COUNT_MAXIMUM][MAX_MULTI_FINGER_TYPE] = { 215 { 216 TouchExplorationMsg::TWO_FINGER_LONG_PRESS_MSG, 217 TouchExplorationMsg::THREE_FINGER_LONG_PRESS_MSG, 218 TouchExplorationMsg::FOUR_FINGER_LONG_PRESS_MSG, 219 }, 220 { 221 TouchExplorationMsg::TWO_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 222 TouchExplorationMsg::THREE_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 223 TouchExplorationMsg::FOUR_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 224 }, 225 { 226 TouchExplorationMsg::TWO_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 227 TouchExplorationMsg::THREE_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 228 TouchExplorationMsg::FOUR_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 229 } 230 }; 231 232 TouchExploration(); ~TouchExploration()233 ~TouchExploration() {} 234 void StartUp(); 235 236 void DestroyEvents() override; 237 void Clear(); 238 void HoverEventRunner(); 239 bool SendDoubleTapAndLongPressDownEvent(); 240 void ProcessMultiFingerGesture(TouchExplorationMsg msg); 241 void CancelPostEvent(TouchExplorationMsg msg); 242 243 /** 244 * @brief Handle pointer events from previous event stream node. 245 * 246 * @param event the pointer event to be handled. 247 * @return true: the event has been processed and does not need to be passed to the next node; 248 * false: the event is not processed. 249 */ 250 bool OnPointerEvent(MMI::PointerEvent &event) override; 251 252 /* Set current state */ SetCurrentState(TouchExplorationState state)253 inline void SetCurrentState(TouchExplorationState state) 254 { 255 HILOG_INFO("currentState is changed from %{public}d to %{public}d.", currentState_, state); 256 currentState_ = state; 257 } 258 259 /* Get current state */ GetCurrentState()260 inline TouchExplorationState GetCurrentState() 261 { 262 return currentState_; 263 } 264 265 private: 266 static constexpr int32_t SWIPE_UP = 0; 267 static constexpr int32_t SWIPE_DOWN = 1; 268 static constexpr int32_t SWIPE_LEFT = 2; 269 static constexpr int32_t SWIPE_RIGHT = 3; 270 271 // Processing Functions in the State Machine 272 void HandleInitStateDown(MMI::PointerEvent &event); 273 void HandleInitStateUp(MMI::PointerEvent &event); 274 void HandleInitStateMove(MMI::PointerEvent &event); 275 void HandlePassingThroughState(MMI::PointerEvent &event); 276 void HandleInvalidState(MMI::PointerEvent &event); 277 void HandleOneFingerDownStateDown(MMI::PointerEvent &event); 278 void HandleOneFingerDownStateUp(MMI::PointerEvent &event); 279 void HandleOneFingerDownStateMove(MMI::PointerEvent &event); 280 void HandleOneFingerLongPressStateDown(MMI::PointerEvent &event); 281 void HandleOneFingerLongPressStateUp(MMI::PointerEvent &event); 282 void HandleOneFingerLongPressStateMove(MMI::PointerEvent &event); 283 void HandleOneFingerSwipeStateDown(MMI::PointerEvent &event); 284 void HandleOneFingerSwipeStateUp(MMI::PointerEvent &event); 285 void HandleOneFingerSwipeStateMove(MMI::PointerEvent &event); 286 void HandleOneFingerSingleTapStateDown(MMI::PointerEvent &event); 287 void HandleOneFingerSingleTapThenDownStateDown(MMI::PointerEvent &event); 288 void HandleOneFingerSingleTapThenDownStateUp(MMI::PointerEvent &event); 289 void HandleOneFingerSingleTapThenDownStateMove(MMI::PointerEvent &event); 290 void HandleOneFingerDoubleTapAndLongPressState(MMI::PointerEvent &event); 291 void HandleTwoFingersDownStateDown(MMI::PointerEvent &event); 292 void HandleTwoFingersDownStateUp(MMI::PointerEvent &event); 293 void HandleTwoFingersDownStateMove(MMI::PointerEvent &event); 294 void HandleTwoFingersDragStateDown(MMI::PointerEvent &event); 295 void HandleTwoFingersDragStateUp(MMI::PointerEvent &event); 296 void HandleTwoFingersDragStateMove(MMI::PointerEvent &event); 297 void HandleTwoFingersTapStateDown(MMI::PointerEvent &event); 298 void HandleMultiFingersTapStateUp(MMI::PointerEvent &event); 299 void HandleTwoFingersTapStateMove(MMI::PointerEvent &event); 300 void HandleMultiFingersContinueDownStateDown(MMI::PointerEvent &event); 301 void HandleTwoFingersContinueDownStateUp(MMI::PointerEvent &event); 302 void HandleTwoFingersContinueDownStateMove(MMI::PointerEvent &event); 303 void HandleTwoFingersUnknownStateDown(MMI::PointerEvent &event); 304 void HandleTwoFingersUnknownStateUp(MMI::PointerEvent &event); 305 void HandleTwoFingersUnknownStateMove(MMI::PointerEvent &event); 306 void HandleThreeFingersDownStateDown(MMI::PointerEvent &event); 307 void HandleThreeFingersDownStateUp(MMI::PointerEvent &event); 308 void HandleThreeFingersDownStateMove(MMI::PointerEvent &event); 309 void HandleThreeFingersSwipeStateDown(MMI::PointerEvent &event); 310 void HandleThreeFingersSwipeStateUp(MMI::PointerEvent &event); 311 void HandleThreeFingersSwipeStateMove(MMI::PointerEvent &event); 312 void HandleThreeFingersTapStateDown(MMI::PointerEvent &event); 313 void HandleThreeFingersTapStateMove(MMI::PointerEvent &event); 314 void HandleThreeFingersContinueDownStateUp(MMI::PointerEvent &event); 315 void HandleThreeFingersContinueDownStateMove(MMI::PointerEvent &event); 316 void HandleFourFingersDownStateDown(MMI::PointerEvent &event); 317 void HandleFourFingersDownStateUp(MMI::PointerEvent &event); 318 void HandleFourFingersDownStateMove(MMI::PointerEvent &event); 319 void HandleFourFingersSwipeStateDown(MMI::PointerEvent &event); 320 void HandleFourFingersSwipeStateUp(MMI::PointerEvent &event); 321 void HandleFourFingersSwipeStateMove(MMI::PointerEvent &event); 322 void HandleFourFingersTapStateDown(MMI::PointerEvent &event); 323 void HandleFourFingersTapStateMove(MMI::PointerEvent &event); 324 void HandleFourFingersContinueDownStateUp(MMI::PointerEvent &event); 325 void HandleFourFingersContinueDownStateMove(MMI::PointerEvent &event); 326 void HandleCancelEvent(MMI::PointerEvent &event); 327 328 void InitOneFingerGestureFuncMap(); 329 void InitTwoFingerGestureFuncMap(); 330 void InitThreeFingerGestureFuncMap(); 331 void InitFourFingerGestureFuncMap(); 332 void HandlePointerEvent(MMI::PointerEvent &event); 333 void AddOneFingerSwipeEvent(MMI::PointerEvent &event); 334 std::vector<Pointer> GetOneFingerSwipePath(); 335 int32_t GetSwipeDirection(const int32_t dx, const int32_t dy); 336 bool RecordFocusedLocation(MMI::PointerEvent &event); 337 void OffsetEvent(MMI::PointerEvent &event); 338 bool GetBasePointItem(MMI::PointerEvent::PointerItem &basePointerIterm, int32_t pId); 339 void GetPointOffset(MMI::PointerEvent &event, std::vector<float> &firstPointOffset, 340 std::vector<float> &secondPointOffset); 341 float GetAngleCos(float offsetX, float offsetY, bool isGetX); 342 bool IsRealMove(MMI::PointerEvent &event); 343 bool IsDragGestureAccept(MMI::PointerEvent &event); 344 void SendAccessibilityEventToAA(EventType eventType); 345 void SendTouchEventToAA(MMI::PointerEvent &event); 346 void SendGestureEventToAA(GestureType gestureId); 347 void SendEventToMultimodal(MMI::PointerEvent event, ChangeAction action); 348 void SendScreenWakeUpEvent(MMI::PointerEvent &event); 349 void SendDragDownEventToMultimodal(MMI::PointerEvent event); 350 void SendUpForDragDownEvent(); 351 bool GetPointerItemWithFingerNum(uint32_t fingerNum, std::vector<MMI::PointerEvent::PointerItem> &curPoints, 352 std::vector<MMI::PointerEvent::PointerItem> &prePoints, MMI::PointerEvent &event); 353 bool IsMultiFingerMultiTap(MMI::PointerEvent &event, const uint32_t fingerNum); 354 bool IsMultiFingerMultiTapGesture(MMI::PointerEvent &event, const uint32_t fingerNum); 355 void HandleMultiFingersTapStateDown(MMI::PointerEvent &event, uint32_t fingerNum); 356 void HandleMultiFingersTapStateMove(MMI::PointerEvent &event, uint32_t fingerNum); 357 void HandleMultiFingersContinueDownStateUp(MMI::PointerEvent &event, uint32_t fingerNum); 358 void HandleMultiFingersContinueDownStateMove(MMI::PointerEvent &event, uint32_t fingerNum); 359 void StoreMultiFingerSwipeBaseDownPoint(); 360 bool GetMultiFingerSwipeBasePointerItem(MMI::PointerEvent::PointerItem &basePointerIterm, int32_t pId); 361 bool SaveMultiFingerSwipeGesturePointerInfo(MMI::PointerEvent &event); 362 bool RecognizeMultiFingerSwipePath(const std::vector<Pointer> &path); 363 GestureType GetMultiFingerSwipeGestureId(uint32_t fingerNum); 364 void HandleMultiFingersSwipeStateUp(MMI::PointerEvent &event, uint32_t fingerNum); 365 std::map<TouchExplorationMsg, GestureType> GetMultiFingerMsgToGestureMap(); 366 void CancelMultiFingerTapEvent(); 367 void CancelMultiFingerTapAndHoldEvent(); 368 CalculateMoveThreshold(int dpi)369 inline float CalculateMoveThreshold(int dpi) 370 { 371 return dpi * COMPLEX_UNIT_MM_CONVERSION * MM_PER_CM; 372 } 373 374 std::shared_ptr<TouchExplorationEventHandler> handler_ = nullptr; 375 std::shared_ptr<AppExecFwk::EventRunner> runner_ = nullptr; 376 std::shared_ptr<AppExecFwk::EventHandler> gestureHandler_ = nullptr; 377 using HandleEventFunc = std::function<void(MMI::PointerEvent &)>; 378 std::map<TouchExplorationState, std::map<int32_t, HandleEventFunc>> handleEventFuncMap_ {}; 379 380 TouchExplorationState currentState_ = TouchExplorationState::TOUCH_INIT; 381 std::list<MMI::PointerEvent> receivedPointerEvents_ {}; 382 383 // single-finger gesture 384 int32_t offsetX_ = 0; 385 int32_t offsetY_ = 0; 386 int32_t focusedWindowId_ = -1; 387 float moveThreshold_ = 0; 388 float xMinPixels_ = 0; 389 float yMinPixels_ = 0; 390 std::vector<Pointer> oneFingerSwipeRoute_ {}; 391 MMI::PointerEvent::PointerItem oneFingerSwipePrePointer_ {}; 392 393 // multi-finger gesture 394 int32_t draggingPid_ = -1; 395 std::shared_ptr<MMI::PointerEvent> draggingDownEvent_ = nullptr; 396 int32_t multiTapNum_ = 0; 397 int32_t multiTapOffsetThresh_ = 0; 398 int32_t multiFingerSwipeDirection_ = -1; 399 float mMinPixelsBetweenSamplesX_ = 0; 400 float mMinPixelsBetweenSamplesY_ = 0; 401 std::map<int32_t, std::vector<Pointer>> multiFingerSwipeRoute_ {}; 402 std::map<int32_t, std::shared_ptr<MMI::PointerEvent>> multiFingerSwipePrePoint_ {}; 403 }; 404 } // namespace Accessibility 405 } // namespace OHOS 406 #endif // ACCESSIBILITY_TOUCH_EXPLORATION_H