1 /* 2 * Copyright (C) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef ACCESSIBILITY_TOUCH_EXPLORATION_H 17 #define ACCESSIBILITY_TOUCH_EXPLORATION_H 18 19 #include <string> 20 #include <cmath> 21 #include <vector> 22 #include <map> 23 #include <functional> 24 #include "accessibility_element_info.h" 25 #include "accessibility_element_operator_callback_stub.h" 26 #include "accessibility_event_transmission.h" 27 #include "accessible_ability_manager_service.h" 28 #include "accessibility_def.h" 29 #include "event_handler.h" 30 #include "hilog_wrapper.h" 31 32 namespace OHOS { 33 namespace Accessibility { 34 class TouchExploration; 35 36 const uint32_t MAX_MULTI_FINGER_TYPE = 3; 37 const int32_t LIMIT_SIZE_TWO = 2; 38 const int32_t LIMIT_SIZE_THREE = 3; 39 const int32_t DIRECTION_NUM = 4; 40 const int32_t TAP_COUNT_MAXIMUM = 3; 41 const int32_t MINI_POINTER_DISTANCE_DIP = 200; 42 const float DEGREES_THRESHOLD = 0.0f; 43 const uint32_t MIN_MULTI_FINGER_SWIPE_POINTER_NUM = 2; 44 const float TOUCH_SLOP = 8.0f; 45 const float MULTI_TAP_SLOP = 100.0f; 46 const float MULTI_TAP_SLOP_DELTA = 0.5f; 47 const int32_t SCREEN_AXIS_NUM = 2; 48 const double MAX_DRAG_GESTURE_COSINE = 0.525321989; 49 const int32_t MM_PER_CM = 10; 50 const double EPSINON = 0.01; 51 const float PIXEL_MULTIPLIER = 0.1f; 52 const int32_t DIVIDE_NUM = 2; 53 const uint32_t FIND_FOCUS_TIMEOUT = 50; 54 const float COMPLEX_UNIT_MM_CONVERSION = 1.0f / 25.4f; 55 const int32_t SIMULATE_POINTER_ID = 10000; 56 #define BIND(func) [this](MMI::PointerEvent& event) { (func(event)); } 57 58 /** 59 * @brief touch exploration state define 60 */ 61 enum class TouchExplorationState : int32_t { 62 TOUCH_INIT, 63 PASSING_THROUGH, 64 INVALID, 65 66 ONE_FINGER_DOWN, 67 ONE_FINGER_LONG_PRESS, 68 ONE_FINGER_SWIPE, 69 ONE_FINGER_SINGLE_TAP, 70 ONE_FINGER_SINGLE_TAP_THEN_DOWN, 71 ONE_FINGER_DOUBLE_TAP_AND_LONG_PRESS, 72 73 TWO_FINGERS_DOWN, 74 TWO_FINGERS_DRAG, 75 TWO_FINGERS_TAP, 76 TWO_FINGERS_CONTINUE_DOWN, 77 TWO_FINGERS_UNKNOWN, 78 79 THREE_FINGERS_DOWN, 80 THREE_FINGERS_SWIPE, 81 THREE_FINGERS_TAP, 82 THREE_FINGERS_CONTINUE_DOWN, 83 84 FOUR_FINGERS_DOWN, 85 FOUR_FINGERS_SWIPE, 86 FOUR_FINGERS_TAP, 87 FOUR_FINGERS_CONTINUE_DOWN 88 }; 89 90 enum class ChangeAction : int32_t { 91 NO_CHANGE, 92 HOVER_MOVE, 93 POINTER_DOWN, 94 POINTER_UP, 95 HOVER_ENTER, 96 HOVER_EXIT, 97 HOVER_CANCEL 98 }; 99 100 enum class PointerCount : uint32_t { 101 POINTER_COUNT_1 = 1, 102 POINTER_COUNT_2 = 2, 103 POINTER_COUNT_3 = 3, 104 POINTER_COUNT_4 = 4 105 }; 106 107 enum class TimeoutDuration : int64_t { 108 LONG_PRESS_TIMEOUT = 200, 109 DOUBLE_TAP_TIMEOUT = 300, 110 MULTI_FINGER_TAP_INTERVAL_TIMEOUT = 100, 111 SWIPE_COMPLETE_TIMEOUT = 300 112 }; 113 114 enum class TouchExplorationMsg : uint32_t { 115 SEND_HOVER_MSG, 116 LONG_PRESS_MSG, 117 DOUBLE_TAP_AND_LONG_PRESS_MSG, 118 SWIPE_COMPLETE_TIMEOUT_MSG, 119 TWO_FINGER_SINGLE_TAP_MSG, 120 TWO_FINGER_LONG_PRESS_MSG, 121 TWO_FINGER_DOUBLE_TAP_MSG, 122 TWO_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 123 TWO_FINGER_TRIPLE_TAP_MSG, 124 TWO_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 125 THREE_FINGER_SINGLE_TAP_MSG, 126 THREE_FINGER_LONG_PRESS_MSG, 127 THREE_FINGER_DOUBLE_TAP_MSG, 128 THREE_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 129 THREE_FINGER_TRIPLE_TAP_MSG, 130 THREE_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 131 FOUR_FINGER_SINGLE_TAP_MSG, 132 FOUR_FINGER_LONG_PRESS_MSG, 133 FOUR_FINGER_DOUBLE_TAP_MSG, 134 FOUR_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 135 FOUR_FINGER_TRIPLE_TAP_MSG, 136 FOUR_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 137 WAIT_ANOTHER_FINGER_DOWN_MSG 138 }; 139 140 struct Pointer { 141 float px_; 142 float py_; 143 }; 144 145 class TouchExplorationEventHandler : public AppExecFwk::EventHandler { 146 public: 147 TouchExplorationEventHandler(const std::shared_ptr<AppExecFwk::EventRunner> &runner, 148 TouchExploration &tgServer); 149 virtual ~TouchExplorationEventHandler() = default; 150 /** 151 * @brief Process the event of install system bundles. 152 * @param event Indicates the event to be processed. 153 */ 154 virtual void ProcessEvent(const AppExecFwk::InnerEvent::Pointer &event) override; 155 156 private: 157 TouchExploration &server_; 158 }; 159 160 class TouchExploration : public EventTransmission { 161 public: 162 static constexpr GestureType GESTURE_DIRECTION[DIRECTION_NUM] = { 163 GestureType::GESTURE_SWIPE_UP, 164 GestureType::GESTURE_SWIPE_DOWN, 165 GestureType::GESTURE_SWIPE_LEFT, 166 GestureType::GESTURE_SWIPE_RIGHT 167 }; 168 169 static constexpr GestureType GESTURE_DIRECTION_TO_ID[DIRECTION_NUM][DIRECTION_NUM] = { 170 { 171 GestureType::GESTURE_SWIPE_UP, 172 GestureType::GESTURE_SWIPE_UP_THEN_DOWN, 173 GestureType::GESTURE_SWIPE_UP_THEN_LEFT, 174 GestureType::GESTURE_SWIPE_UP_THEN_RIGHT, 175 }, 176 { 177 GestureType::GESTURE_SWIPE_DOWN_THEN_UP, 178 GestureType::GESTURE_SWIPE_DOWN, 179 GestureType::GESTURE_SWIPE_DOWN_THEN_LEFT, 180 GestureType::GESTURE_SWIPE_DOWN_THEN_RIGHT, 181 182 }, 183 { 184 GestureType::GESTURE_SWIPE_LEFT_THEN_UP, 185 GestureType::GESTURE_SWIPE_LEFT_THEN_DOWN, 186 GestureType::GESTURE_SWIPE_LEFT, 187 GestureType::GESTURE_SWIPE_LEFT_THEN_RIGHT, 188 189 }, 190 { 191 GestureType::GESTURE_SWIPE_RIGHT_THEN_UP, 192 GestureType::GESTURE_SWIPE_RIGHT_THEN_DOWN, 193 GestureType::GESTURE_SWIPE_RIGHT_THEN_LEFT, 194 GestureType::GESTURE_SWIPE_RIGHT 195 } 196 }; 197 198 static constexpr TouchExplorationMsg GESTURE_TAP_MSG[TAP_COUNT_MAXIMUM][MAX_MULTI_FINGER_TYPE] = { 199 { 200 TouchExplorationMsg::TWO_FINGER_SINGLE_TAP_MSG, 201 TouchExplorationMsg::THREE_FINGER_SINGLE_TAP_MSG, 202 TouchExplorationMsg::FOUR_FINGER_SINGLE_TAP_MSG, 203 }, 204 { 205 TouchExplorationMsg::TWO_FINGER_DOUBLE_TAP_MSG, 206 TouchExplorationMsg::THREE_FINGER_DOUBLE_TAP_MSG, 207 TouchExplorationMsg::FOUR_FINGER_DOUBLE_TAP_MSG, 208 }, 209 { 210 TouchExplorationMsg::TWO_FINGER_TRIPLE_TAP_MSG, 211 TouchExplorationMsg::THREE_FINGER_TRIPLE_TAP_MSG, 212 TouchExplorationMsg::FOUR_FINGER_TRIPLE_TAP_MSG, 213 } 214 }; 215 216 static constexpr TouchExplorationMsg GESTURE_HOLD_MSG[TAP_COUNT_MAXIMUM][MAX_MULTI_FINGER_TYPE] = { 217 { 218 TouchExplorationMsg::TWO_FINGER_LONG_PRESS_MSG, 219 TouchExplorationMsg::THREE_FINGER_LONG_PRESS_MSG, 220 TouchExplorationMsg::FOUR_FINGER_LONG_PRESS_MSG, 221 }, 222 { 223 TouchExplorationMsg::TWO_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 224 TouchExplorationMsg::THREE_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 225 TouchExplorationMsg::FOUR_FINGER_DOUBLE_TAP_AND_HOLD_MSG, 226 }, 227 { 228 TouchExplorationMsg::TWO_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 229 TouchExplorationMsg::THREE_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 230 TouchExplorationMsg::FOUR_FINGER_TRIPLE_TAP_AND_HOLD_MSG, 231 } 232 }; 233 234 TouchExploration(); ~TouchExploration()235 ~TouchExploration() {} 236 void StartUp(); 237 238 void Clear(); 239 void HoverEventRunner(); 240 bool SendDoubleTapAndLongPressDownEvent(); 241 void ProcessMultiFingerGesture(TouchExplorationMsg msg); 242 void CancelPostEvent(TouchExplorationMsg msg); 243 244 /** 245 * @brief Handle pointer events from previous event stream node. 246 * 247 * @param event the pointer event to be handled. 248 * @return true: the event has been processed and does not need to be passed to the next node; 249 * false: the event is not processed. 250 */ 251 bool OnPointerEvent(MMI::PointerEvent &event) override; 252 253 /* Set current state */ SetCurrentState(TouchExplorationState state)254 inline void SetCurrentState(TouchExplorationState state) 255 { 256 HILOG_INFO("currentState is changed from %{public}d to %{public}d.", currentState_, state); 257 currentState_ = state; 258 } 259 260 /* Get current state */ GetCurrentState()261 inline TouchExplorationState GetCurrentState() 262 { 263 return currentState_; 264 } 265 266 private: 267 static constexpr int32_t SWIPE_UP = 0; 268 static constexpr int32_t SWIPE_DOWN = 1; 269 static constexpr int32_t SWIPE_LEFT = 2; 270 static constexpr int32_t SWIPE_RIGHT = 3; 271 272 // Processing Functions in the State Machine 273 void HandleInitStateDown(MMI::PointerEvent &event); 274 void HandleInitStateUp(MMI::PointerEvent &event); 275 void HandleInitStateMove(MMI::PointerEvent &event); 276 void HandlePassingThroughState(MMI::PointerEvent &event); 277 void HandleInvalidState(MMI::PointerEvent &event); 278 void HandleOneFingerDownStateDown(MMI::PointerEvent &event); 279 void HandleOneFingerDownStateUp(MMI::PointerEvent &event); 280 void HandleOneFingerDownStateMove(MMI::PointerEvent &event); 281 void HandleOneFingerLongPressStateDown(MMI::PointerEvent &event); 282 void HandleOneFingerLongPressStateUp(MMI::PointerEvent &event); 283 void HandleOneFingerLongPressStateMove(MMI::PointerEvent &event); 284 void HandleOneFingerSwipeStateDown(MMI::PointerEvent &event); 285 void HandleOneFingerSwipeStateUp(MMI::PointerEvent &event); 286 void HandleOneFingerSwipeStateMove(MMI::PointerEvent &event); 287 void HandleOneFingerSingleTapStateDown(MMI::PointerEvent &event); 288 void HandleOneFingerSingleTapThenDownStateDown(MMI::PointerEvent &event); 289 void HandleOneFingerSingleTapThenDownStateUp(MMI::PointerEvent &event); 290 void HandleOneFingerSingleTapThenDownStateMove(MMI::PointerEvent &event); 291 void HandleOneFingerDoubleTapAndLongPressState(MMI::PointerEvent &event); 292 void HandleTwoFingersDownStateDown(MMI::PointerEvent &event); 293 void HandleTwoFingersDownStateUp(MMI::PointerEvent &event); 294 void HandleTwoFingersDownStateMove(MMI::PointerEvent &event); 295 void HandleTwoFingersDragStateDown(MMI::PointerEvent &event); 296 void HandleTwoFingersDragStateUp(MMI::PointerEvent &event); 297 void HandleTwoFingersDragStateMove(MMI::PointerEvent &event); 298 void HandleTwoFingersTapStateDown(MMI::PointerEvent &event); 299 void HandleMultiFingersTapStateUp(MMI::PointerEvent &event); 300 void HandleTwoFingersTapStateMove(MMI::PointerEvent &event); 301 void HandleMultiFingersContinueDownStateDown(MMI::PointerEvent &event); 302 void HandleTwoFingersContinueDownStateUp(MMI::PointerEvent &event); 303 void HandleTwoFingersContinueDownStateMove(MMI::PointerEvent &event); 304 void HandleTwoFingersUnknownStateDown(MMI::PointerEvent &event); 305 void HandleTwoFingersUnknownStateUp(MMI::PointerEvent &event); 306 void HandleTwoFingersUnknownStateMove(MMI::PointerEvent &event); 307 void HandleThreeFingersDownStateDown(MMI::PointerEvent &event); 308 void HandleThreeFingersDownStateUp(MMI::PointerEvent &event); 309 void HandleThreeFingersDownStateMove(MMI::PointerEvent &event); 310 void HandleThreeFingersSwipeStateDown(MMI::PointerEvent &event); 311 void HandleThreeFingersSwipeStateUp(MMI::PointerEvent &event); 312 void HandleThreeFingersSwipeStateMove(MMI::PointerEvent &event); 313 void HandleThreeFingersTapStateDown(MMI::PointerEvent &event); 314 void HandleThreeFingersTapStateMove(MMI::PointerEvent &event); 315 void HandleThreeFingersContinueDownStateUp(MMI::PointerEvent &event); 316 void HandleThreeFingersContinueDownStateMove(MMI::PointerEvent &event); 317 void HandleFourFingersDownStateDown(MMI::PointerEvent &event); 318 void HandleFourFingersDownStateUp(MMI::PointerEvent &event); 319 void HandleFourFingersDownStateMove(MMI::PointerEvent &event); 320 void HandleFourFingersSwipeStateDown(MMI::PointerEvent &event); 321 void HandleFourFingersSwipeStateUp(MMI::PointerEvent &event); 322 void HandleFourFingersSwipeStateMove(MMI::PointerEvent &event); 323 void HandleFourFingersTapStateDown(MMI::PointerEvent &event); 324 void HandleFourFingersTapStateMove(MMI::PointerEvent &event); 325 void HandleFourFingersContinueDownStateUp(MMI::PointerEvent &event); 326 void HandleFourFingersContinueDownStateMove(MMI::PointerEvent &event); 327 void HandleCancelEvent(MMI::PointerEvent &event); 328 329 void InitOneFingerGestureFuncMap(); 330 void InitTwoFingerGestureFuncMap(); 331 void InitThreeFingerGestureFuncMap(); 332 void InitFourFingerGestureFuncMap(); 333 void HandlePointerEvent(MMI::PointerEvent &event); 334 void AddOneFingerSwipeEvent(MMI::PointerEvent &event); 335 std::vector<Pointer> GetOneFingerSwipePath(); 336 int32_t GetSwipeDirection(const int32_t dx, const int32_t dy); 337 bool RecordFocusedLocation(MMI::PointerEvent &event); 338 void OffsetEvent(MMI::PointerEvent &event); 339 bool GetBasePointItem(MMI::PointerEvent::PointerItem &basePointerItem, int32_t pId); 340 void GetPointOffset(MMI::PointerEvent &event, std::vector<float> &firstPointOffset, 341 std::vector<float> &secondPointOffset); 342 float GetAngleCos(float offsetX, float offsetY, bool isGetX); 343 bool IsRealMove(MMI::PointerEvent &event); 344 bool IsDragGestureAccept(MMI::PointerEvent &event); 345 void SendAccessibilityEventToAA(EventType eventType); 346 void SendTouchEventToAA(MMI::PointerEvent &event); 347 void SendGestureEventToAA(GestureType gestureId); 348 void SendEventToMultimodal(MMI::PointerEvent event, ChangeAction action); 349 void SendScreenWakeUpEvent(MMI::PointerEvent &event); 350 void SendDragDownEventToMultimodal(MMI::PointerEvent event); 351 void SendUpForDragDownEvent(); 352 bool GetPointerItemWithFingerNum(uint32_t fingerNum, std::vector<MMI::PointerEvent::PointerItem> &curPoints, 353 std::vector<MMI::PointerEvent::PointerItem> &prePoints, MMI::PointerEvent &event); 354 bool IsMultiFingerMultiTap(MMI::PointerEvent &event, const uint32_t fingerNum); 355 bool IsMultiFingerMultiTapGesture(MMI::PointerEvent &event, const uint32_t fingerNum); 356 void HandleMultiFingersTapStateDown(MMI::PointerEvent &event, uint32_t fingerNum); 357 void HandleMultiFingersTapStateMove(MMI::PointerEvent &event, uint32_t fingerNum); 358 void HandleMultiFingersContinueDownStateUp(MMI::PointerEvent &event, uint32_t fingerNum); 359 void HandleMultiFingersContinueDownStateMove(MMI::PointerEvent &event, uint32_t fingerNum); 360 void StoreMultiFingerSwipeBaseDownPoint(); 361 bool GetMultiFingerSwipeBasePointerItem(MMI::PointerEvent::PointerItem &basePointerItem, int32_t pId); 362 bool SaveMultiFingerSwipeGesturePointerInfo(MMI::PointerEvent &event); 363 bool RecognizeMultiFingerSwipePath(const std::vector<Pointer> &path); 364 GestureType GetMultiFingerSwipeGestureId(uint32_t fingerNum); 365 void HandleMultiFingersSwipeStateUp(MMI::PointerEvent &event, uint32_t fingerNum); 366 std::map<TouchExplorationMsg, GestureType> GetMultiFingerMsgToGestureMap(); 367 void CancelMultiFingerTapEvent(); 368 void CancelMultiFingerTapAndHoldEvent(); 369 CalculateMoveThreshold(int dpi)370 inline float CalculateMoveThreshold(int dpi) 371 { 372 return dpi * COMPLEX_UNIT_MM_CONVERSION * MM_PER_CM; 373 } 374 375 std::shared_ptr<TouchExplorationEventHandler> handler_ = nullptr; 376 std::shared_ptr<AppExecFwk::EventRunner> runner_ = nullptr; 377 std::shared_ptr<AppExecFwk::EventHandler> gestureHandler_ = nullptr; 378 using HandleEventFunc = std::function<void(MMI::PointerEvent &)>; 379 std::map<TouchExplorationState, std::map<int32_t, HandleEventFunc>> handleEventFuncMap_ {}; 380 381 TouchExplorationState currentState_ = TouchExplorationState::TOUCH_INIT; 382 std::list<MMI::PointerEvent> receivedPointerEvents_ {}; 383 384 // single-finger gesture 385 int32_t offsetX_ = 0; 386 int32_t offsetY_ = 0; 387 float moveThreshold_ = 0; 388 float xMinPixels_ = 0; 389 float yMinPixels_ = 0; 390 std::vector<Pointer> oneFingerSwipeRoute_ {}; 391 MMI::PointerEvent::PointerItem oneFingerSwipePrePointer_ {}; 392 393 // multi-finger gesture 394 int32_t draggingPid_ = -1; 395 std::shared_ptr<MMI::PointerEvent> draggingDownEvent_ = nullptr; 396 int32_t multiTapNum_ = 0; 397 int32_t multiTapOffsetThresh_ = 0; 398 int32_t multiFingerSwipeDirection_ = -1; 399 float mMinPixelsBetweenSamplesX_ = 0; 400 float mMinPixelsBetweenSamplesY_ = 0; 401 std::map<int32_t, std::vector<Pointer>> multiFingerSwipeRoute_ {}; 402 std::map<int32_t, std::shared_ptr<MMI::PointerEvent>> multiFingerSwipePrePoint_ {}; 403 }; 404 } // namespace Accessibility 405 } // namespace OHOS 406 #endif // ACCESSIBILITY_TOUCH_EXPLORATION_H