1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.view; 18 19 import static com.android.text.flags.Flags.handwritingCursorPosition; 20 import static com.android.text.flags.Flags.handwritingTrackDisabled; 21 import static com.android.text.flags.Flags.handwritingUnsupportedMessage; 22 import static com.android.text.flags.Flags.handwritingUnsupportedShowSoftInputFix; 23 24 import android.annotation.FlaggedApi; 25 import android.annotation.NonNull; 26 import android.annotation.Nullable; 27 import android.content.Context; 28 import android.graphics.Matrix; 29 import android.graphics.Rect; 30 import android.graphics.RectF; 31 import android.graphics.Region; 32 import android.text.TextUtils; 33 import android.view.inputmethod.ConnectionlessHandwritingCallback; 34 import android.view.inputmethod.CursorAnchorInfo; 35 import android.view.inputmethod.Flags; 36 import android.view.inputmethod.InputMethodManager; 37 import android.widget.EditText; 38 import android.widget.Editor; 39 import android.widget.TextView; 40 import android.widget.Toast; 41 42 import com.android.internal.R; 43 import com.android.internal.annotations.VisibleForTesting; 44 45 import java.lang.ref.WeakReference; 46 import java.util.ArrayList; 47 import java.util.Iterator; 48 import java.util.List; 49 import java.util.function.Consumer; 50 51 /** 52 * Initiates handwriting mode once it detects stylus movement in handwritable areas. 53 * 54 * It is designed to be used by {@link ViewRootImpl}. For every stylus related MotionEvent that is 55 * dispatched to view tree, ViewRootImpl should call {@link #onTouchEvent} method of this class. 56 * And it will automatically request to enter the handwriting mode when the conditions meet. 57 * 58 * Notice that ViewRootImpl should still dispatch MotionEvents to view tree as usual. 59 * And if it successfully enters the handwriting mode, the ongoing MotionEvent stream will be 60 * routed to the input method. Input system will fabricate an ACTION_CANCEL and send to 61 * ViewRootImpl. 62 * 63 * This class does nothing if: 64 * a) MotionEvents are not from stylus. 65 * b) The user taps or long-clicks with a stylus etc. 66 * c) Stylus pointer down position is not within a handwritable area. 67 * 68 * Used by InputMethodManager. 69 * @hide 70 */ 71 public class HandwritingInitiator { 72 /** 73 * The maximum amount of distance a stylus touch can wander before it is considered 74 * handwriting. 75 */ 76 private final int mHandwritingSlop; 77 /** 78 * The timeout used to distinguish tap or long click from handwriting. If the stylus doesn't 79 * move before this timeout, it's not considered as handwriting. 80 */ 81 private final long mHandwritingTimeoutInMillis; 82 83 private State mState; 84 private final HandwritingAreaTracker mHandwritingAreasTracker = new HandwritingAreaTracker(); 85 86 /** The reference to the View that currently has the input connection. */ 87 @Nullable 88 @VisibleForTesting 89 public WeakReference<View> mConnectedView = null; 90 91 /** 92 * When InputConnection restarts for a View, View#onInputConnectionCreatedInternal 93 * might be called before View#onInputConnectionClosedInternal, so we need to count the input 94 * connections and only set mConnectedView to null when mConnectionCount is zero. 95 */ 96 private int mConnectionCount = 0; 97 98 /** 99 * The reference to the View that currently has focus. 100 * This replaces mConnecteView when {@code Flags#intitiationWithoutInputConnection()} is 101 * enabled. 102 */ 103 @Nullable 104 @VisibleForTesting 105 public WeakReference<View> mFocusedView = null; 106 107 private final InputMethodManager mImm; 108 109 private final int[] mTempLocation = new int[2]; 110 111 private final Rect mTempRect = new Rect(); 112 113 private final RectF mTempRectF = new RectF(); 114 115 private final Region mTempRegion = new Region(); 116 117 private final Matrix mTempMatrix = new Matrix(); 118 119 /** 120 * The handwrite-able View that is currently the target of a hovering stylus pointer. This is 121 * used to help determine whether the handwriting PointerIcon should be shown in 122 * {@link #onResolvePointerIcon(Context, MotionEvent)} so that we can reduce the number of calls 123 * to {@link #findBestCandidateView(float, float, boolean)}. 124 */ 125 @Nullable 126 private WeakReference<View> mCachedHoverTarget = null; 127 128 /** 129 * Whether to show the hover icon for the current connected view. 130 * Hover icon should be hidden for the current connected view after handwriting is initiated 131 * for it until one of the following events happens: 132 * a) user performs a click or long click. In other words, if it receives a series of motion 133 * events that don't trigger handwriting, show hover icon again. 134 * b) the stylus hovers on another editor that supports handwriting (or a handwriting delegate). 135 * c) the current connected editor lost focus. 136 * 137 * If the stylus is hovering on an unconnected editor that supports handwriting, we always show 138 * the hover icon. 139 * TODO(b/308827131): Rename to FocusedView after Flag is flipped. 140 */ 141 private boolean mShowHoverIconForConnectedView = true; 142 143 /** When flag is enabled, touched editors don't wait for InputConnection for initiation. 144 * However, delegation still waits for InputConnection. 145 */ 146 private final boolean mInitiateWithoutConnection = Flags.initiationWithoutInputConnection(); 147 148 @VisibleForTesting HandwritingInitiator(@onNull ViewConfiguration viewConfiguration, @NonNull InputMethodManager inputMethodManager)149 public HandwritingInitiator(@NonNull ViewConfiguration viewConfiguration, 150 @NonNull InputMethodManager inputMethodManager) { 151 mHandwritingSlop = viewConfiguration.getScaledHandwritingSlop(); 152 mHandwritingTimeoutInMillis = ViewConfiguration.getLongPressTimeout(); 153 mImm = inputMethodManager; 154 } 155 156 /** 157 * Notify the HandwritingInitiator that a new MotionEvent has arrived. 158 * 159 * <p>The return value indicates whether the event has been fully handled by the 160 * HandwritingInitiator and should not be dispatched to the view tree. This will be true for 161 * ACTION_MOVE events from a stylus gesture after handwriting mode has been initiated, in order 162 * to suppress other actions such as scrolling. 163 * 164 * <p>If HandwritingInitiator triggers the handwriting mode, a fabricated ACTION_CANCEL event 165 * will be sent to the ViewRootImpl. 166 * 167 * @param motionEvent the stylus {@link MotionEvent} 168 * @return true if the event has been fully handled by the {@link HandwritingInitiator} and 169 * should not be dispatched to the {@link View} tree, or false if the event should be dispatched 170 * to the {@link View} tree as usual 171 */ 172 @VisibleForTesting onTouchEvent(@onNull MotionEvent motionEvent)173 public boolean onTouchEvent(@NonNull MotionEvent motionEvent) { 174 final int maskedAction = motionEvent.getActionMasked(); 175 switch (maskedAction) { 176 case MotionEvent.ACTION_DOWN: 177 case MotionEvent.ACTION_POINTER_DOWN: 178 mState = null; 179 if (!motionEvent.isStylusPointer()) { 180 // The motion event is not from a stylus event, ignore it. 181 return false; 182 } 183 mState = new State(motionEvent); 184 break; 185 case MotionEvent.ACTION_POINTER_UP: 186 final int pointerId = motionEvent.getPointerId(motionEvent.getActionIndex()); 187 if (mState == null || pointerId != mState.mStylusPointerId) { 188 // ACTION_POINTER_UP is from another stylus pointer, ignore the event. 189 return false; 190 } 191 // Deliberately fall through. 192 case MotionEvent.ACTION_CANCEL: 193 case MotionEvent.ACTION_UP: 194 // If it's ACTION_CANCEL or ACTION_UP, all the pointers go up. There is no need to 195 // check whether the stylus we are tracking goes up. 196 if (mState != null) { 197 mState.mShouldInitHandwriting = false; 198 if (!mState.mHandled) { 199 // The user just did a click, long click or another stylus gesture, 200 // show hover icon again for the connected view. 201 mShowHoverIconForConnectedView = true; 202 } 203 } 204 return false; 205 case MotionEvent.ACTION_MOVE: 206 if (mState == null) { 207 return false; 208 } 209 210 // Either we've already tried to initiate handwriting, or the ongoing MotionEvent 211 // sequence is considered to be tap, long-click or other gestures. 212 if (!mState.mShouldInitHandwriting || mState.mExceedHandwritingSlop) { 213 return mState.mHandled; 214 } 215 216 final long timeElapsed = 217 motionEvent.getEventTime() - mState.mStylusDownTimeInMillis; 218 if (timeElapsed > mHandwritingTimeoutInMillis) { 219 mState.mShouldInitHandwriting = false; 220 return mState.mHandled; 221 } 222 223 final int pointerIndex = motionEvent.findPointerIndex(mState.mStylusPointerId); 224 final float x = motionEvent.getX(pointerIndex); 225 final float y = motionEvent.getY(pointerIndex); 226 if (largerThanTouchSlop(x, y, mState.mStylusDownX, mState.mStylusDownY)) { 227 mState.mExceedHandwritingSlop = true; 228 View candidateView = findBestCandidateView(mState.mStylusDownX, 229 mState.mStylusDownY, /* isHover */ false); 230 if (candidateView != null && candidateView.isEnabled()) { 231 boolean candidateHasFocus = candidateView.hasFocus(); 232 if (!candidateView.isStylusHandwritingAvailable()) { 233 mState.mShouldInitHandwriting = false; 234 return false; 235 } else if (shouldShowHandwritingUnavailableMessageForView(candidateView)) { 236 int messagesResId = (candidateView instanceof TextView tv 237 && tv.isAnyPasswordInputType()) 238 ? R.string.error_handwriting_unsupported_password 239 : R.string.error_handwriting_unsupported; 240 Toast.makeText(candidateView.getContext(), messagesResId, 241 Toast.LENGTH_SHORT).show(); 242 if (!candidateView.hasFocus()) { 243 requestFocusWithoutReveal(candidateView); 244 } 245 if (!handwritingUnsupportedShowSoftInputFix() 246 || (candidateView instanceof TextView tv 247 && tv.getShowSoftInputOnFocus())) { 248 mImm.showSoftInput(candidateView, 0); 249 } 250 mState.mHandled = true; 251 mState.mShouldInitHandwriting = false; 252 motionEvent.setAction((motionEvent.getAction() 253 & MotionEvent.ACTION_POINTER_INDEX_MASK) 254 | MotionEvent.ACTION_CANCEL); 255 candidateView.getRootView().dispatchTouchEvent(motionEvent); 256 } else if (candidateView == getConnectedOrFocusedView()) { 257 if (!candidateHasFocus) { 258 requestFocusWithoutReveal(candidateView); 259 } 260 startHandwriting(candidateView); 261 } else if (candidateView.getHandwritingDelegatorCallback() != null) { 262 prepareDelegation(candidateView); 263 } else { 264 if (mInitiateWithoutConnection) { 265 if (!candidateHasFocus) { 266 // schedule for view focus. 267 mState.mPendingFocusedView = new WeakReference<>(candidateView); 268 requestFocusWithoutReveal(candidateView); 269 } 270 } else { 271 mState.mPendingConnectedView = new WeakReference<>(candidateView); 272 if (!candidateHasFocus) { 273 requestFocusWithoutReveal(candidateView); 274 } 275 } 276 } 277 } 278 } 279 return mState.mHandled; 280 } 281 return false; 282 } 283 284 @Nullable getConnectedView()285 private View getConnectedView() { 286 if (mConnectedView == null) return null; 287 return mConnectedView.get(); 288 } 289 clearConnectedView()290 private void clearConnectedView() { 291 mConnectedView = null; 292 mConnectionCount = 0; 293 } 294 295 /** 296 * Notify HandwritingInitiator that a delegate view (see {@link View#isHandwritingDelegate}) 297 * gained focus. 298 */ onDelegateViewFocused(@onNull View view)299 public void onDelegateViewFocused(@NonNull View view) { 300 if (mInitiateWithoutConnection) { 301 onEditorFocused(view); 302 } 303 if (view == getConnectedView()) { 304 tryAcceptStylusHandwritingDelegation(view); 305 } 306 } 307 308 /** 309 * Notify HandwritingInitiator that a new InputConnection is created. 310 * The caller of this method should guarantee that each onInputConnectionCreated call 311 * is paired with a onInputConnectionClosed call. 312 * @param view the view that created the current InputConnection. 313 * @see #onInputConnectionClosed(View) 314 */ onInputConnectionCreated(@onNull View view)315 public void onInputConnectionCreated(@NonNull View view) { 316 if (mInitiateWithoutConnection && !view.isHandwritingDelegate()) { 317 // When flag is enabled, only delegation continues to wait for InputConnection. 318 return; 319 } 320 if (!view.isAutoHandwritingEnabled()) { 321 clearConnectedView(); 322 return; 323 } 324 325 final View connectedView = getConnectedView(); 326 if (connectedView == view) { 327 ++mConnectionCount; 328 } else { 329 mConnectedView = new WeakReference<>(view); 330 mConnectionCount = 1; 331 // A new view just gain focus. By default, we should show hover icon for it. 332 mShowHoverIconForConnectedView = true; 333 if (view.isHandwritingDelegate() && tryAcceptStylusHandwritingDelegation(view)) { 334 // tryAcceptStylusHandwritingDelegation should set boolean below, however, we 335 // cannot mock IMM to return true for acceptStylusDelegation(). 336 // TODO(b/324670412): we should move any dependent tests to integration and remove 337 // the assignment below. 338 mShowHoverIconForConnectedView = false; 339 return; 340 } 341 if (!mInitiateWithoutConnection && mState != null 342 && mState.mPendingConnectedView != null 343 && mState.mPendingConnectedView.get() == view) { 344 startHandwriting(view); 345 } 346 } 347 } 348 349 /** 350 * Notify HandwritingInitiator that a new editor is focused. 351 * @param view the view that received focus. 352 */ 353 @VisibleForTesting onEditorFocused(@onNull View view)354 public void onEditorFocused(@NonNull View view) { 355 if (!mInitiateWithoutConnection) { 356 return; 357 } 358 359 final View focusedView = getFocusedView(); 360 361 if (!handwritingTrackDisabled() && !view.isAutoHandwritingEnabled()) { 362 clearFocusedView(focusedView); 363 return; 364 } 365 366 if (focusedView == view) { 367 return; 368 } 369 updateFocusedView(view); 370 371 if (mState != null && mState.mPendingFocusedView != null 372 && mState.mPendingFocusedView.get() == view 373 && (!handwritingTrackDisabled() || view.isAutoHandwritingEnabled())) { 374 startHandwriting(view); 375 } 376 } 377 378 /** 379 * Notify HandwritingInitiator that the InputConnection has closed for the given view. 380 * The caller of this method should guarantee that each onInputConnectionClosed call 381 * is paired with a onInputConnectionCreated call. 382 * @param view the view that closed the InputConnection. 383 */ onInputConnectionClosed(@onNull View view)384 public void onInputConnectionClosed(@NonNull View view) { 385 if (mInitiateWithoutConnection && !view.isHandwritingDelegate()) { 386 return; 387 } 388 final View connectedView = getConnectedView(); 389 if (connectedView == null) return; 390 if (connectedView == view) { 391 --mConnectionCount; 392 if (mConnectionCount == 0) { 393 clearConnectedView(); 394 } 395 } else { 396 // Unexpected branch, set mConnectedView to null to avoid further problem. 397 clearConnectedView(); 398 } 399 } 400 401 @Nullable getFocusedView()402 private View getFocusedView() { 403 if (mFocusedView == null) return null; 404 return mFocusedView.get(); 405 } 406 407 /** 408 * Clear the tracked focused view tracked for handwriting initiation. 409 * @param view the focused view. 410 */ clearFocusedView(View view)411 public void clearFocusedView(View view) { 412 if (view == null || mFocusedView == null) { 413 return; 414 } 415 if (mFocusedView.get() == view) { 416 mFocusedView = null; 417 } 418 } 419 420 /** 421 * Called when new {@link Editor} is focused. 422 * @return {@code true} if handwriting can initiate for given view. 423 */ 424 @VisibleForTesting updateFocusedView(@onNull View view)425 public boolean updateFocusedView(@NonNull View view) { 426 if (!handwritingTrackDisabled() && !view.shouldInitiateHandwriting()) { 427 mFocusedView = null; 428 return false; 429 } 430 431 final View focusedView = getFocusedView(); 432 if (focusedView != view) { 433 mFocusedView = new WeakReference<>(view); 434 if (!handwritingTrackDisabled() || view.shouldInitiateHandwriting()) { 435 // A new view just gain focus. By default, we should show hover icon for it. 436 mShowHoverIconForConnectedView = true; 437 } 438 } 439 440 return true; 441 } 442 443 /** Starts a stylus handwriting session for the view. */ 444 @VisibleForTesting startHandwriting(@onNull View view)445 public void startHandwriting(@NonNull View view) { 446 mImm.startStylusHandwriting(view); 447 mState.mHandled = true; 448 mState.mShouldInitHandwriting = false; 449 mShowHoverIconForConnectedView = false; 450 if (view instanceof TextView) { 451 ((TextView) view).hideHint(); 452 } 453 } 454 prepareDelegation(View view)455 private void prepareDelegation(View view) { 456 String delegatePackageName = view.getAllowedHandwritingDelegatePackageName(); 457 if (delegatePackageName == null) { 458 delegatePackageName = view.getContext().getOpPackageName(); 459 } 460 if (mImm.isConnectionlessStylusHandwritingAvailable()) { 461 // No other view should have focus during the connectionless handwriting session, as 462 // this could cause user confusion about the input target for the session. 463 view.getViewRootImpl().getView().clearFocus(); 464 mImm.startConnectionlessStylusHandwritingForDelegation( 465 view, getCursorAnchorInfoForConnectionless(view), delegatePackageName, 466 view::post, new DelegationCallback(view, delegatePackageName)); 467 mState.mShouldInitHandwriting = false; 468 } else { 469 mImm.prepareStylusHandwritingDelegation(view, delegatePackageName); 470 view.getHandwritingDelegatorCallback().run(); 471 } 472 mState.mHandled = true; 473 } 474 475 /** 476 * Starts a stylus handwriting session for the delegate view, if {@link 477 * InputMethodManager#prepareStylusHandwritingDelegation} was previously called. 478 */ 479 @VisibleForTesting tryAcceptStylusHandwritingDelegation(@onNull View view)480 public boolean tryAcceptStylusHandwritingDelegation(@NonNull View view) { 481 if (Flags.useZeroJankProxy()) { 482 tryAcceptStylusHandwritingDelegationAsync(view); 483 } else { 484 return tryAcceptStylusHandwritingDelegationInternal(view); 485 } 486 return false; 487 } 488 tryAcceptStylusHandwritingDelegationInternal(@onNull View view)489 private boolean tryAcceptStylusHandwritingDelegationInternal(@NonNull View view) { 490 String delegatorPackageName = 491 view.getAllowedHandwritingDelegatorPackageName(); 492 if (delegatorPackageName == null) { 493 delegatorPackageName = view.getContext().getOpPackageName(); 494 } 495 if (mImm.acceptStylusHandwritingDelegation(view, delegatorPackageName)) { 496 onDelegationAccepted(view); 497 return true; 498 } 499 return false; 500 } 501 502 @FlaggedApi(Flags.FLAG_USE_ZERO_JANK_PROXY) tryAcceptStylusHandwritingDelegationAsync(@onNull View view)503 private void tryAcceptStylusHandwritingDelegationAsync(@NonNull View view) { 504 String delegatorPackageName = 505 view.getAllowedHandwritingDelegatorPackageName(); 506 if (delegatorPackageName == null) { 507 delegatorPackageName = view.getContext().getOpPackageName(); 508 } 509 WeakReference<View> viewRef = new WeakReference<>(view); 510 Consumer<Boolean> consumer = delegationAccepted -> { 511 if (delegationAccepted) { 512 onDelegationAccepted(viewRef.get()); 513 } 514 }; 515 mImm.acceptStylusHandwritingDelegation(view, delegatorPackageName, view::post, consumer); 516 } 517 onDelegationAccepted(View view)518 private void onDelegationAccepted(View view) { 519 if (mState != null) { 520 mState.mHandled = true; 521 mState.mShouldInitHandwriting = false; 522 } 523 if (view == null) { 524 // can be null if view was detached and was GCed. 525 return; 526 } 527 if (view instanceof TextView) { 528 ((TextView) view).hideHint(); 529 } 530 // A handwriting delegate view is accepted and handwriting starts; hide the 531 // hover icon. 532 mShowHoverIconForConnectedView = false; 533 } 534 535 /** 536 * Notify that the handwriting area for the given view might be updated. 537 * @param view the view whose handwriting area might be updated. 538 */ updateHandwritingAreasForView(@onNull View view)539 public void updateHandwritingAreasForView(@NonNull View view) { 540 mHandwritingAreasTracker.updateHandwritingAreaForView(view); 541 } 542 shouldTriggerStylusHandwritingForView(@onNull View view)543 private static boolean shouldTriggerStylusHandwritingForView(@NonNull View view) { 544 if (!view.shouldInitiateHandwriting()) { 545 return false; 546 } 547 // The view may be a handwriting initiation delegator, in which case it is not the editor 548 // view for which handwriting would be started. However, in almost all cases, the return 549 // values of View#isStylusHandwritingAvailable will be the same for the delegator view and 550 // the delegate editor view. So the delegator view can be used to decide whether handwriting 551 // should be triggered. 552 return view.isStylusHandwritingAvailable(); 553 } 554 shouldShowHandwritingUnavailableMessageForView(@onNull View view)555 private static boolean shouldShowHandwritingUnavailableMessageForView(@NonNull View view) { 556 return (view instanceof TextView) && !shouldTriggerStylusHandwritingForView(view); 557 } 558 shouldTriggerHandwritingOrShowUnavailableMessageForView( @onNull View view)559 private static boolean shouldTriggerHandwritingOrShowUnavailableMessageForView( 560 @NonNull View view) { 561 return (view instanceof TextView) || shouldTriggerStylusHandwritingForView(view); 562 } 563 564 /** 565 * Returns the pointer icon for the motion event, or null if it doesn't specify the icon. 566 * This gives HandwritingInitiator a chance to show the stylus handwriting icon over a 567 * handwrite-able area. 568 */ onResolvePointerIcon(Context context, MotionEvent event)569 public PointerIcon onResolvePointerIcon(Context context, MotionEvent event) { 570 final View hoverView = findHoverView(event); 571 if (hoverView == null || !shouldTriggerStylusHandwritingForView(hoverView)) { 572 return null; 573 } 574 575 if (mShowHoverIconForConnectedView) { 576 return PointerIcon.getSystemIcon(context, PointerIcon.TYPE_HANDWRITING); 577 } 578 579 if (hoverView != getConnectedOrFocusedView()) { 580 // The stylus is hovering on another view that supports handwriting. We should show 581 // hover icon. Also reset the mShowHoverIconForFocusedView so that hover 582 // icon is displayed again next time when the stylus hovers on focused view. 583 mShowHoverIconForConnectedView = true; 584 return PointerIcon.getSystemIcon(context, PointerIcon.TYPE_HANDWRITING); 585 } 586 return null; 587 } 588 589 // TODO(b/308827131): Remove once Flag is flipped. getConnectedOrFocusedView()590 private View getConnectedOrFocusedView() { 591 if (mInitiateWithoutConnection) { 592 return mFocusedView == null ? null : mFocusedView.get(); 593 } else { 594 return mConnectedView == null ? null : mConnectedView.get(); 595 } 596 } 597 getCachedHoverTarget()598 private View getCachedHoverTarget() { 599 if (mCachedHoverTarget == null) { 600 return null; 601 } 602 return mCachedHoverTarget.get(); 603 } 604 findHoverView(MotionEvent event)605 private View findHoverView(MotionEvent event) { 606 if (!event.isStylusPointer() || !event.isHoverEvent()) { 607 return null; 608 } 609 610 if (event.getActionMasked() == MotionEvent.ACTION_HOVER_ENTER 611 || event.getActionMasked() == MotionEvent.ACTION_HOVER_MOVE) { 612 final float hoverX = event.getX(event.getActionIndex()); 613 final float hoverY = event.getY(event.getActionIndex()); 614 615 final View cachedHoverTarget = getCachedHoverTarget(); 616 if (cachedHoverTarget != null) { 617 final Rect handwritingArea = mTempRect; 618 if (getViewHandwritingArea(cachedHoverTarget, handwritingArea) 619 && isInHandwritingArea(handwritingArea, hoverX, hoverY, cachedHoverTarget, 620 /* isHover */ true) 621 && shouldTriggerStylusHandwritingForView(cachedHoverTarget)) { 622 return cachedHoverTarget; 623 } 624 } 625 626 final View candidateView = findBestCandidateView(hoverX, hoverY, /* isHover */ true); 627 628 if (candidateView != null) { 629 if (!handwritingUnsupportedMessage()) { 630 mCachedHoverTarget = new WeakReference<>(candidateView); 631 } 632 return candidateView; 633 } 634 } 635 636 mCachedHoverTarget = null; 637 return null; 638 } 639 requestFocusWithoutReveal(View view)640 private void requestFocusWithoutReveal(View view) { 641 if (!handwritingCursorPosition() && view instanceof EditText editText 642 && !mState.mStylusDownWithinEditorBounds) { 643 // If the stylus down point was inside the EditText's bounds, then the EditText will 644 // automatically set its cursor position nearest to the stylus down point when it 645 // gains focus. If the stylus down point was outside the EditText's bounds (within 646 // the extended handwriting bounds), then we must calculate and set the cursor 647 // position manually. 648 view.getLocationInWindow(mTempLocation); 649 int offset = editText.getOffsetForPosition( 650 mState.mStylusDownX - mTempLocation[0], 651 mState.mStylusDownY - mTempLocation[1]); 652 editText.setSelection(offset); 653 } 654 if (view.getRevealOnFocusHint()) { 655 view.setRevealOnFocusHint(false); 656 view.requestFocus(); 657 view.setRevealOnFocusHint(true); 658 } else { 659 view.requestFocus(); 660 } 661 if (handwritingCursorPosition() && view instanceof EditText editText) { 662 // Move the cursor to the end of the paragraph closest to the stylus down point. 663 view.getLocationInWindow(mTempLocation); 664 int line = editText.getLineAtCoordinate(mState.mStylusDownY - mTempLocation[1]); 665 int paragraphEnd = TextUtils.indexOf(editText.getText(), '\n', 666 editText.getLayout().getLineStart(line)); 667 if (paragraphEnd < 0) { 668 paragraphEnd = editText.getText().length(); 669 } 670 editText.setSelection(paragraphEnd); 671 } 672 } 673 674 /** 675 * Given the location of the stylus event, return the best candidate view to initialize 676 * handwriting mode or show the handwriting unavailable error message. 677 * 678 * @param x the x coordinates of the stylus event, in the coordinates of the window. 679 * @param y the y coordinates of the stylus event, in the coordinates of the window. 680 */ 681 @Nullable findBestCandidateView(float x, float y, boolean isHover)682 private View findBestCandidateView(float x, float y, boolean isHover) { 683 // TODO(b/308827131): Rename to FocusedView after Flag is flipped. 684 // If the connectedView is not null and do not set any handwriting area, it will check 685 // whether the connectedView's boundary contains the initial stylus position. If true, 686 // directly return the connectedView. 687 final View connectedOrFocusedView = getConnectedOrFocusedView(); 688 if (connectedOrFocusedView != null) { 689 Rect handwritingArea = mTempRect; 690 if (getViewHandwritingArea(connectedOrFocusedView, handwritingArea) 691 && isInHandwritingArea(handwritingArea, x, y, connectedOrFocusedView, isHover) 692 && shouldTriggerHandwritingOrShowUnavailableMessageForView( 693 connectedOrFocusedView)) { 694 if (!isHover && mState != null) { 695 mState.mStylusDownWithinEditorBounds = 696 contains(handwritingArea, x, y, 0f, 0f, 0f, 0f); 697 } 698 return connectedOrFocusedView; 699 } 700 } 701 702 float minDistance = Float.MAX_VALUE; 703 View bestCandidate = null; 704 // Check the registered handwriting areas. 705 final List<HandwritableViewInfo> handwritableViewInfos = 706 mHandwritingAreasTracker.computeViewInfos(); 707 for (HandwritableViewInfo viewInfo : handwritableViewInfos) { 708 final View view = viewInfo.getView(); 709 final Rect handwritingArea = viewInfo.getHandwritingArea(); 710 if (!isInHandwritingArea(handwritingArea, x, y, view, isHover) 711 || !shouldTriggerHandwritingOrShowUnavailableMessageForView(view)) { 712 continue; 713 } 714 715 final float distance = distance(handwritingArea, x, y); 716 if (distance == 0f) { 717 if (!isHover && mState != null) { 718 mState.mStylusDownWithinEditorBounds = true; 719 } 720 return view; 721 } 722 if (distance < minDistance) { 723 minDistance = distance; 724 bestCandidate = view; 725 } 726 } 727 return bestCandidate; 728 } 729 730 /** 731 * Return the square of the distance from point (x, y) to the given rect, which is mainly used 732 * for comparison. The distance is defined to be: the shortest distance between (x, y) to any 733 * point on rect. When (x, y) is contained by the rect, return 0f. 734 */ distance(@onNull Rect rect, float x, float y)735 private static float distance(@NonNull Rect rect, float x, float y) { 736 if (contains(rect, x, y, 0f, 0f, 0f, 0f)) { 737 return 0f; 738 } 739 740 /* The distance between point (x, y) and rect, there are 2 basic cases: 741 * a) The distance is the distance from (x, y) to the closest corner on rect. 742 * o | | 743 * ---+-----+--- 744 * | | 745 * ---+-----+--- 746 * | | 747 * b) The distance is the distance from (x, y) to the closest edge on rect. 748 * | o | 749 * ---+-----+--- 750 * | | 751 * ---+-----+--- 752 * | | 753 * We define xDistance as following(similar for yDistance): 754 * If x is in [left, right) 0, else min(abs(x - left), abs(x - y)) 755 * For case a, sqrt(xDistance^2 + yDistance^2) is the final distance. 756 * For case b, distance should be yDistance, which is also equal to 757 * sqrt(xDistance^2 + yDistance^2) because xDistance is 0. 758 */ 759 final float xDistance; 760 if (x >= rect.left && x < rect.right) { 761 xDistance = 0f; 762 } else if (x < rect.left) { 763 xDistance = rect.left - x; 764 } else { 765 xDistance = x - rect.right; 766 } 767 768 final float yDistance; 769 if (y >= rect.top && y < rect.bottom) { 770 yDistance = 0f; 771 } else if (y < rect.top) { 772 yDistance = rect.top - y; 773 } else { 774 yDistance = y - rect.bottom; 775 } 776 // We can omit sqrt here because we only need the distance for comparison. 777 return xDistance * xDistance + yDistance * yDistance; 778 } 779 780 /** 781 * Return the handwriting area of the given view, represented in the window's coordinate. 782 * If the view didn't set any handwriting area, it will return the view's boundary. 783 * 784 * <p> The handwriting area is clipped to its visible part. 785 * Notice that the returned rectangle is the view's original handwriting area without the 786 * view's handwriting area extends. </p> 787 * 788 * @param view the {@link View} whose handwriting area we want to compute. 789 * @param rect the {@link Rect} to receive the result. 790 * 791 * @return true if the view's handwriting area is still visible, or false if it's clipped and 792 * fully invisible. This method only consider the clip by given view's parents, but not the case 793 * where a view is covered by its sibling view. 794 */ getViewHandwritingArea(@onNull View view, @NonNull Rect rect)795 private static boolean getViewHandwritingArea(@NonNull View view, @NonNull Rect rect) { 796 final ViewParent viewParent = view.getParent(); 797 if (viewParent != null && view.isAttachedToWindow() && view.isAggregatedVisible()) { 798 final Rect localHandwritingArea = view.getHandwritingArea(); 799 if (localHandwritingArea != null) { 800 rect.set(localHandwritingArea); 801 } else { 802 rect.set(0, 0, view.getWidth(), view.getHeight()); 803 } 804 return viewParent.getChildVisibleRect(view, rect, null); 805 } 806 return false; 807 } 808 809 /** 810 * Return true if the (x, y) is inside by the given {@link Rect} with the View's 811 * handwriting bounds with offsets applied. 812 */ isInHandwritingArea(@ullable Rect handwritingArea, float x, float y, View view, boolean isHover)813 private boolean isInHandwritingArea(@Nullable Rect handwritingArea, 814 float x, float y, View view, boolean isHover) { 815 if (handwritingArea == null) return false; 816 817 if (!contains(handwritingArea, x, y, 818 view.getHandwritingBoundsOffsetLeft(), 819 view.getHandwritingBoundsOffsetTop(), 820 view.getHandwritingBoundsOffsetRight(), 821 view.getHandwritingBoundsOffsetBottom())) { 822 return false; 823 } 824 825 // The returned handwritingArea computed by ViewParent#getChildVisibleRect didn't consider 826 // the case where a view is stacking on top of the editor. (e.g. DrawerLayout, popup) 827 // We must check the hit region of the editor again, and avoid the case where another 828 // view on top of the editor is handling MotionEvents. 829 ViewParent parent = view.getParent(); 830 if (parent == null) { 831 return true; 832 } 833 834 Region region = mTempRegion; 835 mTempRegion.set(0, 0, view.getWidth(), view.getHeight()); 836 Matrix matrix = mTempMatrix; 837 matrix.reset(); 838 if (!parent.getChildLocalHitRegion(view, region, matrix, isHover)) { 839 return false; 840 } 841 842 // It's not easy to extend the region by the given handwritingBoundsOffset. Instead, we 843 // create a rectangle surrounding the motion event location and check if this rectangle 844 // overlaps with the hit region of the editor. 845 float left = x - view.getHandwritingBoundsOffsetRight(); 846 float top = y - view.getHandwritingBoundsOffsetBottom(); 847 float right = Math.max(x + view.getHandwritingBoundsOffsetLeft(), left + 1); 848 float bottom = Math.max(y + view.getHandwritingBoundsOffsetTop(), top + 1); 849 RectF rectF = mTempRectF; 850 rectF.set(left, top, right, bottom); 851 matrix.mapRect(rectF); 852 853 return region.op(Math.round(rectF.left), Math.round(rectF.top), 854 Math.round(rectF.right), Math.round(rectF.bottom), Region.Op.INTERSECT); 855 } 856 857 /** 858 * Return true if the (x, y) is inside by the given {@link Rect} offset by the given 859 * offsetLeft, offsetTop, offsetRight and offsetBottom. 860 */ contains(@onNull Rect rect, float x, float y, float offsetLeft, float offsetTop, float offsetRight, float offsetBottom)861 private static boolean contains(@NonNull Rect rect, float x, float y, 862 float offsetLeft, float offsetTop, float offsetRight, float offsetBottom) { 863 return x >= rect.left - offsetLeft && x < rect.right + offsetRight 864 && y >= rect.top - offsetTop && y < rect.bottom + offsetBottom; 865 } 866 largerThanTouchSlop(float x1, float y1, float x2, float y2)867 private boolean largerThanTouchSlop(float x1, float y1, float x2, float y2) { 868 float dx = x1 - x2; 869 float dy = y1 - y2; 870 return dx * dx + dy * dy > mHandwritingSlop * mHandwritingSlop; 871 } 872 873 /** Object that keeps the MotionEvent related states for HandwritingInitiator. */ 874 private static class State { 875 /** 876 * Whether it should initiate handwriting mode for the current MotionEvent sequence. 877 * (A series of MotionEvents from ACTION_DOWN to ACTION_UP) 878 * 879 * The purpose of this boolean value is: 880 * a) We should only request to start handwriting mode ONCE for each MotionEvent sequence. 881 * If we've already requested to enter handwriting mode for the ongoing MotionEvent 882 * sequence, this boolean is set to false. And it won't request to start handwriting again. 883 * 884 * b) If the MotionEvent sequence is considered to be tap, long-click or other gestures. 885 * This boolean will be set to false, and it won't request to start handwriting. 886 */ 887 private boolean mShouldInitHandwriting; 888 889 /** 890 * Whether the current MotionEvent sequence has been handled by the handwriting initiator, 891 * either by initiating handwriting mode, or by preparing handwriting delegation. 892 */ 893 private boolean mHandled; 894 895 /** 896 * Whether the current ongoing stylus MotionEvent sequence already exceeds the 897 * handwriting slop. 898 * It's used for the case where the stylus exceeds handwriting slop before the target View 899 * built InputConnection. 900 */ 901 private boolean mExceedHandwritingSlop; 902 903 /** 904 * Whether the stylus down point of the MotionEvent sequence was within the editor's bounds 905 * (not including the extended handwriting bounds). 906 */ 907 private boolean mStylusDownWithinEditorBounds; 908 909 /** 910 * A view which has requested focus and is pending input connection creation. When an input 911 * connection is created for the view, a handwriting session should be started for the view. 912 */ 913 private WeakReference<View> mPendingConnectedView = null; 914 915 /** 916 * A view which has requested focus and is yet to receive it. 917 * When view receives focus, a handwriting session should be started for the view. 918 */ 919 private WeakReference<View> mPendingFocusedView = null; 920 921 /** The pointer id of the stylus pointer that is being tracked. */ 922 private final int mStylusPointerId; 923 /** The time stamp when the stylus pointer goes down. */ 924 private final long mStylusDownTimeInMillis; 925 /** The initial location where the stylus pointer goes down. */ 926 private final float mStylusDownX; 927 private final float mStylusDownY; 928 State(MotionEvent motionEvent)929 private State(MotionEvent motionEvent) { 930 final int actionIndex = motionEvent.getActionIndex(); 931 mStylusPointerId = motionEvent.getPointerId(actionIndex); 932 mStylusDownTimeInMillis = motionEvent.getEventTime(); 933 mStylusDownX = motionEvent.getX(actionIndex); 934 mStylusDownY = motionEvent.getY(actionIndex); 935 936 mShouldInitHandwriting = true; 937 mHandled = false; 938 mExceedHandwritingSlop = false; 939 } 940 } 941 942 /** The helper method to check if the given view is still active for handwriting. */ isViewActive(@ullable View view)943 private static boolean isViewActive(@Nullable View view) { 944 return view != null && view.isAttachedToWindow() && view.isAggregatedVisible() 945 && view.shouldTrackHandwritingArea(); 946 } 947 getCursorAnchorInfoForConnectionless(View view)948 private CursorAnchorInfo getCursorAnchorInfoForConnectionless(View view) { 949 CursorAnchorInfo.Builder builder = new CursorAnchorInfo.Builder(); 950 // Fake editor views will usually display hint text. The hint text view can be used to 951 // populate the CursorAnchorInfo. 952 TextView textView = findFirstTextViewDescendent(view); 953 if (textView != null) { 954 textView.getCursorAnchorInfo(0, builder, mTempMatrix); 955 if (textView.getSelectionStart() < 0) { 956 // Insertion marker location is not populated if selection start is negative, so 957 // make a best guess. 958 float bottom = textView.getHeight() - textView.getExtendedPaddingBottom(); 959 builder.setInsertionMarkerLocation( 960 /* horizontalPosition= */ textView.getCompoundPaddingStart(), 961 /* lineTop= */ textView.getExtendedPaddingTop(), 962 /* lineBaseline= */ bottom, 963 /* lineBottom= */ bottom, 964 /* flags= */ 0); 965 } 966 } else { 967 // If there is no TextView descendent, just populate the insertion marker with the start 968 // edge of the view. 969 mTempMatrix.reset(); 970 view.transformMatrixToGlobal(mTempMatrix); 971 builder.setMatrix(mTempMatrix); 972 builder.setInsertionMarkerLocation( 973 /* horizontalPosition= */ view.isLayoutRtl() ? view.getWidth() : 0, 974 /* lineTop= */ 0, 975 /* lineBaseline= */ view.getHeight(), 976 /* lineBottom= */ view.getHeight(), 977 /* flags= */ 0); 978 } 979 return builder.build(); 980 } 981 982 @Nullable findFirstTextViewDescendent(View view)983 private static TextView findFirstTextViewDescendent(View view) { 984 if (view instanceof ViewGroup viewGroup) { 985 TextView textView; 986 for (int i = 0; i < viewGroup.getChildCount(); ++i) { 987 View child = viewGroup.getChildAt(i); 988 textView = (child instanceof TextView tv) 989 ? tv : findFirstTextViewDescendent(viewGroup.getChildAt(i)); 990 if (textView != null 991 && textView.isAggregatedVisible() 992 && (!TextUtils.isEmpty(textView.getText()) 993 || !TextUtils.isEmpty(textView.getHint()))) { 994 return textView; 995 } 996 } 997 } 998 return null; 999 } 1000 1001 /** 1002 * A class used to track the handwriting areas set by the Views. 1003 * 1004 * @hide 1005 */ 1006 @VisibleForTesting 1007 public static class HandwritingAreaTracker { 1008 private final List<HandwritableViewInfo> mHandwritableViewInfos; 1009 HandwritingAreaTracker()1010 public HandwritingAreaTracker() { 1011 mHandwritableViewInfos = new ArrayList<>(); 1012 } 1013 1014 /** 1015 * Notify this tracker that the handwriting area of the given view has been updated. 1016 * This method does three things: 1017 * a) iterate over the all the tracked ViewInfos and remove those already invalid ones. 1018 * b) mark the given view's ViewInfo to be dirty. So that next time when 1019 * {@link #computeViewInfos} is called, this view's handwriting area will be recomputed. 1020 * c) If no the given view is not in the tracked ViewInfo list, a new ViewInfo object will 1021 * be created and added to the list. 1022 * 1023 * @param view the view whose handwriting area is updated. 1024 */ updateHandwritingAreaForView(@onNull View view)1025 public void updateHandwritingAreaForView(@NonNull View view) { 1026 Iterator<HandwritableViewInfo> iterator = mHandwritableViewInfos.iterator(); 1027 boolean found = false; 1028 while (iterator.hasNext()) { 1029 final HandwritableViewInfo handwritableViewInfo = iterator.next(); 1030 final View curView = handwritableViewInfo.getView(); 1031 if (!isViewActive(curView)) { 1032 iterator.remove(); 1033 } 1034 if (curView == view) { 1035 found = true; 1036 handwritableViewInfo.mIsDirty = true; 1037 } 1038 } 1039 if (!found && isViewActive(view)) { 1040 // The given view is not tracked. Create a new HandwritableViewInfo for it and add 1041 // to the list. 1042 mHandwritableViewInfos.add(new HandwritableViewInfo(view)); 1043 } 1044 } 1045 1046 /** 1047 * Update the handwriting areas and return a list of ViewInfos containing the view 1048 * reference and its handwriting area. 1049 */ 1050 @NonNull computeViewInfos()1051 public List<HandwritableViewInfo> computeViewInfos() { 1052 mHandwritableViewInfos.removeIf(viewInfo -> !viewInfo.update()); 1053 return mHandwritableViewInfos; 1054 } 1055 } 1056 1057 /** 1058 * A class that reference to a View and its handwriting area(in the ViewRoot's coordinate.) 1059 * 1060 * @hide 1061 */ 1062 @VisibleForTesting 1063 public static class HandwritableViewInfo { 1064 final WeakReference<View> mViewRef; 1065 Rect mHandwritingArea = null; 1066 @VisibleForTesting 1067 public boolean mIsDirty = true; 1068 1069 @VisibleForTesting HandwritableViewInfo(@onNull View view)1070 public HandwritableViewInfo(@NonNull View view) { 1071 mViewRef = new WeakReference<>(view); 1072 } 1073 1074 /** Return the tracked view. */ 1075 @Nullable getView()1076 public View getView() { 1077 return mViewRef.get(); 1078 } 1079 1080 /** 1081 * Return the tracked handwriting area, represented in the ViewRoot's coordinates. 1082 * Notice, the caller should not modify the returned Rect. 1083 */ 1084 @Nullable getHandwritingArea()1085 public Rect getHandwritingArea() { 1086 return mHandwritingArea; 1087 } 1088 1089 /** 1090 * Update the handwriting area in this ViewInfo. 1091 * 1092 * @return true if this ViewInfo is still valid. Or false if this ViewInfo has become 1093 * invalid due to either view is no longer visible, or the handwriting area set by the 1094 * view is removed. {@link HandwritingAreaTracker} no longer need to keep track of this 1095 * HandwritableViewInfo this method returns false. 1096 */ update()1097 public boolean update() { 1098 final View view = getView(); 1099 if (!isViewActive(view)) { 1100 return false; 1101 } 1102 1103 if (!mIsDirty) { 1104 return true; 1105 } 1106 final Rect handwritingArea = view.getHandwritingArea(); 1107 if (handwritingArea == null) { 1108 return false; 1109 } 1110 1111 ViewParent parent = view.getParent(); 1112 if (parent != null) { 1113 if (mHandwritingArea == null) { 1114 mHandwritingArea = new Rect(); 1115 } 1116 mHandwritingArea.set(handwritingArea); 1117 if (!parent.getChildVisibleRect(view, mHandwritingArea, null /* offset */)) { 1118 mHandwritingArea = null; 1119 } 1120 } 1121 mIsDirty = false; 1122 return true; 1123 } 1124 } 1125 1126 private class DelegationCallback implements ConnectionlessHandwritingCallback { 1127 private final View mView; 1128 private final String mDelegatePackageName; 1129 DelegationCallback(View view, String delegatePackageName)1130 private DelegationCallback(View view, String delegatePackageName) { 1131 mView = view; 1132 mDelegatePackageName = delegatePackageName; 1133 } 1134 1135 @Override onResult(@onNull CharSequence text)1136 public void onResult(@NonNull CharSequence text) { 1137 mView.getHandwritingDelegatorCallback().run(); 1138 } 1139 1140 @Override onError(int errorCode)1141 public void onError(int errorCode) { 1142 switch (errorCode) { 1143 case CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED: 1144 mView.getHandwritingDelegatorCallback().run(); 1145 break; 1146 case CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED: 1147 // Fall back to the old delegation flow 1148 mImm.prepareStylusHandwritingDelegation(mView, mDelegatePackageName); 1149 mView.getHandwritingDelegatorCallback().run(); 1150 break; 1151 } 1152 } 1153 } 1154 } 1155