1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.widget; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.UiThread; 22 import android.annotation.WorkerThread; 23 import android.content.Context; 24 import android.graphics.Canvas; 25 import android.graphics.PointF; 26 import android.graphics.RectF; 27 import android.os.AsyncTask; 28 import android.os.Build; 29 import android.os.LocaleList; 30 import android.text.Layout; 31 import android.text.Selection; 32 import android.text.Spannable; 33 import android.text.TextUtils; 34 import android.text.util.Linkify; 35 import android.util.Log; 36 import android.view.ActionMode; 37 import android.view.textclassifier.SelectionEvent; 38 import android.view.textclassifier.SelectionEvent.InvocationMethod; 39 import android.view.textclassifier.SelectionSessionLogger; 40 import android.view.textclassifier.TextClassification; 41 import android.view.textclassifier.TextClassificationConstants; 42 import android.view.textclassifier.TextClassificationManager; 43 import android.view.textclassifier.TextClassifier; 44 import android.view.textclassifier.TextSelection; 45 import android.widget.Editor.SelectionModifierCursorController; 46 47 import com.android.internal.annotations.VisibleForTesting; 48 import com.android.internal.util.Preconditions; 49 50 import java.text.BreakIterator; 51 import java.util.ArrayList; 52 import java.util.Comparator; 53 import java.util.List; 54 import java.util.Objects; 55 import java.util.function.Consumer; 56 import java.util.function.Function; 57 import java.util.function.Supplier; 58 import java.util.regex.Pattern; 59 60 /** 61 * Helper class for starting selection action mode 62 * (synchronously without the TextClassifier, asynchronously with the TextClassifier). 63 * @hide 64 */ 65 @UiThread 66 @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE) 67 public final class SelectionActionModeHelper { 68 69 private static final String LOG_TAG = "SelectActionModeHelper"; 70 71 private final Editor mEditor; 72 private final TextView mTextView; 73 private final TextClassificationHelper mTextClassificationHelper; 74 75 @Nullable private TextClassification mTextClassification; 76 private AsyncTask mTextClassificationAsyncTask; 77 78 private final SelectionTracker mSelectionTracker; 79 80 // TODO remove nullable marker once the switch gating the feature gets removed 81 @Nullable 82 private final SmartSelectSprite mSmartSelectSprite; 83 SelectionActionModeHelper(@onNull Editor editor)84 SelectionActionModeHelper(@NonNull Editor editor) { 85 mEditor = Preconditions.checkNotNull(editor); 86 mTextView = mEditor.getTextView(); 87 mTextClassificationHelper = new TextClassificationHelper( 88 mTextView.getContext(), 89 mTextView::getTextClassifier, 90 getText(mTextView), 91 0, 1, mTextView.getTextLocales()); 92 mSelectionTracker = new SelectionTracker(mTextView); 93 94 if (getTextClassificationSettings().isSmartSelectionAnimationEnabled()) { 95 mSmartSelectSprite = new SmartSelectSprite(mTextView.getContext(), 96 editor.getTextView().mHighlightColor, mTextView::invalidate); 97 } else { 98 mSmartSelectSprite = null; 99 } 100 } 101 102 /** 103 * Starts Selection ActionMode. 104 */ startSelectionActionModeAsync(boolean adjustSelection)105 public void startSelectionActionModeAsync(boolean adjustSelection) { 106 // Check if the smart selection should run for editable text. 107 adjustSelection &= getTextClassificationSettings().isSmartSelectionEnabled(); 108 109 mSelectionTracker.onOriginalSelection( 110 getText(mTextView), 111 mTextView.getSelectionStart(), 112 mTextView.getSelectionEnd(), 113 false /*isLink*/); 114 cancelAsyncTask(); 115 if (skipTextClassification()) { 116 startSelectionActionMode(null); 117 } else { 118 resetTextClassificationHelper(); 119 mTextClassificationAsyncTask = new TextClassificationAsyncTask( 120 mTextView, 121 mTextClassificationHelper.getTimeoutDuration(), 122 adjustSelection 123 ? mTextClassificationHelper::suggestSelection 124 : mTextClassificationHelper::classifyText, 125 mSmartSelectSprite != null 126 ? this::startSelectionActionModeWithSmartSelectAnimation 127 : this::startSelectionActionMode, 128 mTextClassificationHelper::getOriginalSelection) 129 .execute(); 130 } 131 } 132 133 /** 134 * Starts Link ActionMode. 135 */ startLinkActionModeAsync(int start, int end)136 public void startLinkActionModeAsync(int start, int end) { 137 mSelectionTracker.onOriginalSelection(getText(mTextView), start, end, true /*isLink*/); 138 cancelAsyncTask(); 139 if (skipTextClassification()) { 140 startLinkActionMode(null); 141 } else { 142 resetTextClassificationHelper(start, end); 143 mTextClassificationAsyncTask = new TextClassificationAsyncTask( 144 mTextView, 145 mTextClassificationHelper.getTimeoutDuration(), 146 mTextClassificationHelper::classifyText, 147 this::startLinkActionMode, 148 mTextClassificationHelper::getOriginalSelection) 149 .execute(); 150 } 151 } 152 invalidateActionModeAsync()153 public void invalidateActionModeAsync() { 154 cancelAsyncTask(); 155 if (skipTextClassification()) { 156 invalidateActionMode(null); 157 } else { 158 resetTextClassificationHelper(); 159 mTextClassificationAsyncTask = new TextClassificationAsyncTask( 160 mTextView, 161 mTextClassificationHelper.getTimeoutDuration(), 162 mTextClassificationHelper::classifyText, 163 this::invalidateActionMode, 164 mTextClassificationHelper::getOriginalSelection) 165 .execute(); 166 } 167 } 168 onSelectionAction(int menuItemId)169 public void onSelectionAction(int menuItemId) { 170 mSelectionTracker.onSelectionAction( 171 mTextView.getSelectionStart(), mTextView.getSelectionEnd(), 172 getActionType(menuItemId), mTextClassification); 173 } 174 onSelectionDrag()175 public void onSelectionDrag() { 176 mSelectionTracker.onSelectionAction( 177 mTextView.getSelectionStart(), mTextView.getSelectionEnd(), 178 SelectionEvent.ACTION_DRAG, mTextClassification); 179 } 180 onTextChanged(int start, int end)181 public void onTextChanged(int start, int end) { 182 mSelectionTracker.onTextChanged(start, end, mTextClassification); 183 } 184 resetSelection(int textIndex)185 public boolean resetSelection(int textIndex) { 186 if (mSelectionTracker.resetSelection(textIndex, mEditor)) { 187 invalidateActionModeAsync(); 188 return true; 189 } 190 return false; 191 } 192 193 @Nullable getTextClassification()194 public TextClassification getTextClassification() { 195 return mTextClassification; 196 } 197 onDestroyActionMode()198 public void onDestroyActionMode() { 199 cancelSmartSelectAnimation(); 200 mSelectionTracker.onSelectionDestroyed(); 201 cancelAsyncTask(); 202 } 203 onDraw(final Canvas canvas)204 public void onDraw(final Canvas canvas) { 205 if (isDrawingHighlight() && mSmartSelectSprite != null) { 206 mSmartSelectSprite.draw(canvas); 207 } 208 } 209 isDrawingHighlight()210 public boolean isDrawingHighlight() { 211 return mSmartSelectSprite != null && mSmartSelectSprite.isAnimationActive(); 212 } 213 getTextClassificationSettings()214 private TextClassificationConstants getTextClassificationSettings() { 215 return TextClassificationManager.getSettings(mTextView.getContext()); 216 } 217 cancelAsyncTask()218 private void cancelAsyncTask() { 219 if (mTextClassificationAsyncTask != null) { 220 mTextClassificationAsyncTask.cancel(true); 221 mTextClassificationAsyncTask = null; 222 } 223 mTextClassification = null; 224 } 225 skipTextClassification()226 private boolean skipTextClassification() { 227 // No need to make an async call for a no-op TextClassifier. 228 final boolean noOpTextClassifier = mTextView.usesNoOpTextClassifier(); 229 // Do not call the TextClassifier if there is no selection. 230 final boolean noSelection = mTextView.getSelectionEnd() == mTextView.getSelectionStart(); 231 // Do not call the TextClassifier if this is a password field. 232 final boolean password = mTextView.hasPasswordTransformationMethod() 233 || TextView.isPasswordInputType(mTextView.getInputType()); 234 return noOpTextClassifier || noSelection || password; 235 } 236 startLinkActionMode(@ullable SelectionResult result)237 private void startLinkActionMode(@Nullable SelectionResult result) { 238 startActionMode(Editor.TextActionMode.TEXT_LINK, result); 239 } 240 startSelectionActionMode(@ullable SelectionResult result)241 private void startSelectionActionMode(@Nullable SelectionResult result) { 242 startActionMode(Editor.TextActionMode.SELECTION, result); 243 } 244 startActionMode( @ditor.TextActionMode int actionMode, @Nullable SelectionResult result)245 private void startActionMode( 246 @Editor.TextActionMode int actionMode, @Nullable SelectionResult result) { 247 final CharSequence text = getText(mTextView); 248 if (result != null && text instanceof Spannable 249 && (mTextView.isTextSelectable() || mTextView.isTextEditable())) { 250 // Do not change the selection if TextClassifier should be dark launched. 251 if (!getTextClassificationSettings().isModelDarkLaunchEnabled()) { 252 Selection.setSelection((Spannable) text, result.mStart, result.mEnd); 253 mTextView.invalidate(); 254 } 255 mTextClassification = result.mClassification; 256 } else if (result != null && actionMode == Editor.TextActionMode.TEXT_LINK) { 257 mTextClassification = result.mClassification; 258 } else { 259 mTextClassification = null; 260 } 261 if (mEditor.startActionModeInternal(actionMode)) { 262 final SelectionModifierCursorController controller = mEditor.getSelectionController(); 263 if (controller != null 264 && (mTextView.isTextSelectable() || mTextView.isTextEditable())) { 265 controller.show(); 266 } 267 if (result != null) { 268 switch (actionMode) { 269 case Editor.TextActionMode.SELECTION: 270 mSelectionTracker.onSmartSelection(result); 271 break; 272 case Editor.TextActionMode.TEXT_LINK: 273 mSelectionTracker.onLinkSelected(result); 274 break; 275 default: 276 break; 277 } 278 } 279 } 280 mEditor.setRestartActionModeOnNextRefresh(false); 281 mTextClassificationAsyncTask = null; 282 } 283 startSelectionActionModeWithSmartSelectAnimation( @ullable SelectionResult result)284 private void startSelectionActionModeWithSmartSelectAnimation( 285 @Nullable SelectionResult result) { 286 final Layout layout = mTextView.getLayout(); 287 288 final Runnable onAnimationEndCallback = () -> { 289 final SelectionResult startSelectionResult; 290 if (result != null && result.mStart >= 0 && result.mEnd <= getText(mTextView).length() 291 && result.mStart <= result.mEnd) { 292 startSelectionResult = result; 293 } else { 294 startSelectionResult = null; 295 } 296 startSelectionActionMode(startSelectionResult); 297 }; 298 // TODO do not trigger the animation if the change included only non-printable characters 299 final boolean didSelectionChange = 300 result != null && (mTextView.getSelectionStart() != result.mStart 301 || mTextView.getSelectionEnd() != result.mEnd); 302 303 if (!didSelectionChange) { 304 onAnimationEndCallback.run(); 305 return; 306 } 307 308 final List<SmartSelectSprite.RectangleWithTextSelectionLayout> selectionRectangles = 309 convertSelectionToRectangles(layout, result.mStart, result.mEnd); 310 311 final PointF touchPoint = new PointF( 312 mEditor.getLastUpPositionX(), 313 mEditor.getLastUpPositionY()); 314 315 final PointF animationStartPoint = 316 movePointInsideNearestRectangle(touchPoint, selectionRectangles, 317 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle); 318 319 mSmartSelectSprite.startAnimation( 320 animationStartPoint, 321 selectionRectangles, 322 onAnimationEndCallback); 323 } 324 convertSelectionToRectangles( final Layout layout, final int start, final int end)325 private List<SmartSelectSprite.RectangleWithTextSelectionLayout> convertSelectionToRectangles( 326 final Layout layout, final int start, final int end) { 327 final List<SmartSelectSprite.RectangleWithTextSelectionLayout> result = new ArrayList<>(); 328 329 final Layout.SelectionRectangleConsumer consumer = 330 (left, top, right, bottom, textSelectionLayout) -> mergeRectangleIntoList( 331 result, 332 new RectF(left, top, right, bottom), 333 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle, 334 r -> new SmartSelectSprite.RectangleWithTextSelectionLayout(r, 335 textSelectionLayout) 336 ); 337 338 layout.getSelection(start, end, consumer); 339 340 result.sort(Comparator.comparing( 341 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle, 342 SmartSelectSprite.RECTANGLE_COMPARATOR)); 343 344 return result; 345 } 346 347 // TODO: Move public pure functions out of this class and make it package-private. 348 /** 349 * Merges a {@link RectF} into an existing list of any objects which contain a rectangle. 350 * While merging, this method makes sure that: 351 * 352 * <ol> 353 * <li>No rectangle is redundant (contained within a bigger rectangle)</li> 354 * <li>Rectangles of the same height and vertical position that intersect get merged</li> 355 * </ol> 356 * 357 * @param list the list of rectangles (or other rectangle containers) to merge the new 358 * rectangle into 359 * @param candidate the {@link RectF} to merge into the list 360 * @param extractor a function that can extract a {@link RectF} from an element of the given 361 * list 362 * @param packer a function that can wrap the resulting {@link RectF} into an element that 363 * the list contains 364 * @hide 365 */ 366 @VisibleForTesting mergeRectangleIntoList(final List<T> list, final RectF candidate, final Function<T, RectF> extractor, final Function<RectF, T> packer)367 public static <T> void mergeRectangleIntoList(final List<T> list, 368 final RectF candidate, final Function<T, RectF> extractor, 369 final Function<RectF, T> packer) { 370 if (candidate.isEmpty()) { 371 return; 372 } 373 374 final int elementCount = list.size(); 375 for (int index = 0; index < elementCount; ++index) { 376 final RectF existingRectangle = extractor.apply(list.get(index)); 377 if (existingRectangle.contains(candidate)) { 378 return; 379 } 380 if (candidate.contains(existingRectangle)) { 381 existingRectangle.setEmpty(); 382 continue; 383 } 384 385 final boolean rectanglesContinueEachOther = candidate.left == existingRectangle.right 386 || candidate.right == existingRectangle.left; 387 final boolean canMerge = candidate.top == existingRectangle.top 388 && candidate.bottom == existingRectangle.bottom 389 && (RectF.intersects(candidate, existingRectangle) 390 || rectanglesContinueEachOther); 391 392 if (canMerge) { 393 candidate.union(existingRectangle); 394 existingRectangle.setEmpty(); 395 } 396 } 397 398 for (int index = elementCount - 1; index >= 0; --index) { 399 final RectF rectangle = extractor.apply(list.get(index)); 400 if (rectangle.isEmpty()) { 401 list.remove(index); 402 } 403 } 404 405 list.add(packer.apply(candidate)); 406 } 407 408 409 /** @hide */ 410 @VisibleForTesting movePointInsideNearestRectangle(final PointF point, final List<T> list, final Function<T, RectF> extractor)411 public static <T> PointF movePointInsideNearestRectangle(final PointF point, 412 final List<T> list, final Function<T, RectF> extractor) { 413 float bestX = -1; 414 float bestY = -1; 415 double bestDistance = Double.MAX_VALUE; 416 417 final int elementCount = list.size(); 418 for (int index = 0; index < elementCount; ++index) { 419 final RectF rectangle = extractor.apply(list.get(index)); 420 final float candidateY = rectangle.centerY(); 421 final float candidateX; 422 423 if (point.x > rectangle.right) { 424 candidateX = rectangle.right; 425 } else if (point.x < rectangle.left) { 426 candidateX = rectangle.left; 427 } else { 428 candidateX = point.x; 429 } 430 431 final double candidateDistance = Math.pow(point.x - candidateX, 2) 432 + Math.pow(point.y - candidateY, 2); 433 434 if (candidateDistance < bestDistance) { 435 bestX = candidateX; 436 bestY = candidateY; 437 bestDistance = candidateDistance; 438 } 439 } 440 441 return new PointF(bestX, bestY); 442 } 443 invalidateActionMode(@ullable SelectionResult result)444 private void invalidateActionMode(@Nullable SelectionResult result) { 445 cancelSmartSelectAnimation(); 446 mTextClassification = result != null ? result.mClassification : null; 447 final ActionMode actionMode = mEditor.getTextActionMode(); 448 if (actionMode != null) { 449 actionMode.invalidate(); 450 } 451 mSelectionTracker.onSelectionUpdated( 452 mTextView.getSelectionStart(), mTextView.getSelectionEnd(), mTextClassification); 453 mTextClassificationAsyncTask = null; 454 } 455 resetTextClassificationHelper(int selectionStart, int selectionEnd)456 private void resetTextClassificationHelper(int selectionStart, int selectionEnd) { 457 if (selectionStart < 0 || selectionEnd < 0) { 458 // Use selection indices 459 selectionStart = mTextView.getSelectionStart(); 460 selectionEnd = mTextView.getSelectionEnd(); 461 } 462 mTextClassificationHelper.init( 463 mTextView::getTextClassifier, 464 getText(mTextView), 465 selectionStart, selectionEnd, 466 mTextView.getTextLocales()); 467 } 468 resetTextClassificationHelper()469 private void resetTextClassificationHelper() { 470 resetTextClassificationHelper(-1, -1); 471 } 472 cancelSmartSelectAnimation()473 private void cancelSmartSelectAnimation() { 474 if (mSmartSelectSprite != null) { 475 mSmartSelectSprite.cancelAnimation(); 476 } 477 } 478 479 /** 480 * Tracks and logs smart selection changes. 481 * It is important to trigger this object's methods at the appropriate event so that it tracks 482 * smart selection events appropriately. 483 */ 484 private static final class SelectionTracker { 485 486 private final TextView mTextView; 487 private SelectionMetricsLogger mLogger; 488 489 private int mOriginalStart; 490 private int mOriginalEnd; 491 private int mSelectionStart; 492 private int mSelectionEnd; 493 private boolean mAllowReset; 494 private final LogAbandonRunnable mDelayedLogAbandon = new LogAbandonRunnable(); 495 SelectionTracker(TextView textView)496 SelectionTracker(TextView textView) { 497 mTextView = Preconditions.checkNotNull(textView); 498 mLogger = new SelectionMetricsLogger(textView); 499 } 500 501 /** 502 * Called when the original selection happens, before smart selection is triggered. 503 */ onOriginalSelection( CharSequence text, int selectionStart, int selectionEnd, boolean isLink)504 public void onOriginalSelection( 505 CharSequence text, int selectionStart, int selectionEnd, boolean isLink) { 506 // If we abandoned a selection and created a new one very shortly after, we may still 507 // have a pending request to log ABANDON, which we flush here. 508 mDelayedLogAbandon.flush(); 509 510 mOriginalStart = mSelectionStart = selectionStart; 511 mOriginalEnd = mSelectionEnd = selectionEnd; 512 mAllowReset = false; 513 maybeInvalidateLogger(); 514 mLogger.logSelectionStarted(mTextView.getTextClassificationSession(), 515 text, selectionStart, 516 isLink ? SelectionEvent.INVOCATION_LINK : SelectionEvent.INVOCATION_MANUAL); 517 } 518 519 /** 520 * Called when selection action mode is started and the results come from a classifier. 521 */ onSmartSelection(SelectionResult result)522 public void onSmartSelection(SelectionResult result) { 523 onClassifiedSelection(result); 524 mLogger.logSelectionModified( 525 result.mStart, result.mEnd, result.mClassification, result.mSelection); 526 } 527 528 /** 529 * Called when link action mode is started and the classification comes from a classifier. 530 */ onLinkSelected(SelectionResult result)531 public void onLinkSelected(SelectionResult result) { 532 onClassifiedSelection(result); 533 // TODO: log (b/70246800) 534 } 535 onClassifiedSelection(SelectionResult result)536 private void onClassifiedSelection(SelectionResult result) { 537 if (isSelectionStarted()) { 538 mSelectionStart = result.mStart; 539 mSelectionEnd = result.mEnd; 540 mAllowReset = mSelectionStart != mOriginalStart || mSelectionEnd != mOriginalEnd; 541 } 542 } 543 544 /** 545 * Called when selection bounds change. 546 */ onSelectionUpdated( int selectionStart, int selectionEnd, @Nullable TextClassification classification)547 public void onSelectionUpdated( 548 int selectionStart, int selectionEnd, 549 @Nullable TextClassification classification) { 550 if (isSelectionStarted()) { 551 mSelectionStart = selectionStart; 552 mSelectionEnd = selectionEnd; 553 mAllowReset = false; 554 mLogger.logSelectionModified(selectionStart, selectionEnd, classification, null); 555 } 556 } 557 558 /** 559 * Called when the selection action mode is destroyed. 560 */ onSelectionDestroyed()561 public void onSelectionDestroyed() { 562 mAllowReset = false; 563 // Wait a few ms to see if the selection was destroyed because of a text change event. 564 mDelayedLogAbandon.schedule(100 /* ms */); 565 } 566 567 /** 568 * Called when an action is taken on a smart selection. 569 */ onSelectionAction( int selectionStart, int selectionEnd, @SelectionEvent.ActionType int action, @Nullable TextClassification classification)570 public void onSelectionAction( 571 int selectionStart, int selectionEnd, 572 @SelectionEvent.ActionType int action, 573 @Nullable TextClassification classification) { 574 if (isSelectionStarted()) { 575 mAllowReset = false; 576 mLogger.logSelectionAction(selectionStart, selectionEnd, action, classification); 577 } 578 } 579 580 /** 581 * Returns true if the current smart selection should be reset to normal selection based on 582 * information that has been recorded about the original selection and the smart selection. 583 * The expected UX here is to allow the user to select a word inside of the smart selection 584 * on a single tap. 585 */ resetSelection(int textIndex, Editor editor)586 public boolean resetSelection(int textIndex, Editor editor) { 587 final TextView textView = editor.getTextView(); 588 if (isSelectionStarted() 589 && mAllowReset 590 && textIndex >= mSelectionStart && textIndex <= mSelectionEnd 591 && getText(textView) instanceof Spannable) { 592 mAllowReset = false; 593 boolean selected = editor.selectCurrentWord(); 594 if (selected) { 595 mSelectionStart = editor.getTextView().getSelectionStart(); 596 mSelectionEnd = editor.getTextView().getSelectionEnd(); 597 mLogger.logSelectionAction( 598 textView.getSelectionStart(), textView.getSelectionEnd(), 599 SelectionEvent.ACTION_RESET, null /* classification */); 600 } 601 return selected; 602 } 603 return false; 604 } 605 onTextChanged(int start, int end, TextClassification classification)606 public void onTextChanged(int start, int end, TextClassification classification) { 607 if (isSelectionStarted() && start == mSelectionStart && end == mSelectionEnd) { 608 onSelectionAction(start, end, SelectionEvent.ACTION_OVERTYPE, classification); 609 } 610 } 611 maybeInvalidateLogger()612 private void maybeInvalidateLogger() { 613 if (mLogger.isEditTextLogger() != mTextView.isTextEditable()) { 614 mLogger = new SelectionMetricsLogger(mTextView); 615 } 616 } 617 isSelectionStarted()618 private boolean isSelectionStarted() { 619 return mSelectionStart >= 0 && mSelectionEnd >= 0 && mSelectionStart != mSelectionEnd; 620 } 621 622 /** A helper for keeping track of pending abandon logging requests. */ 623 private final class LogAbandonRunnable implements Runnable { 624 private boolean mIsPending; 625 626 /** Schedules an abandon to be logged with the given delay. Flush if necessary. */ schedule(int delayMillis)627 void schedule(int delayMillis) { 628 if (mIsPending) { 629 Log.e(LOG_TAG, "Force flushing abandon due to new scheduling request"); 630 flush(); 631 } 632 mIsPending = true; 633 mTextView.postDelayed(this, delayMillis); 634 } 635 636 /** If there is a pending log request, execute it now. */ flush()637 void flush() { 638 mTextView.removeCallbacks(this); 639 run(); 640 } 641 642 @Override run()643 public void run() { 644 if (mIsPending) { 645 mLogger.logSelectionAction( 646 mSelectionStart, mSelectionEnd, 647 SelectionEvent.ACTION_ABANDON, null /* classification */); 648 mSelectionStart = mSelectionEnd = -1; 649 mLogger.endTextClassificationSession(); 650 mIsPending = false; 651 } 652 } 653 } 654 } 655 656 // TODO: Write tests 657 /** 658 * Metrics logging helper. 659 * 660 * This logger logs selection by word indices. The initial (start) single word selection is 661 * logged at [0, 1) -- end index is exclusive. Other word indices are logged relative to the 662 * initial single word selection. 663 * e.g. New York city, NY. Suppose the initial selection is "York" in 664 * "New York city, NY", then "York" is at [0, 1), "New" is at [-1, 0], and "city" is at [1, 2). 665 * "New York" is at [-1, 1). 666 * Part selection of a word e.g. "or" is counted as selecting the 667 * entire word i.e. equivalent to "York", and each special character is counted as a word, e.g. 668 * "," is at [2, 3). Whitespaces are ignored. 669 * 670 * NOTE that the definition of a word is defined by the TextClassifier's Logger's token 671 * iterator. 672 */ 673 private static final class SelectionMetricsLogger { 674 675 private static final String LOG_TAG = "SelectionMetricsLogger"; 676 private static final Pattern PATTERN_WHITESPACE = Pattern.compile("\\s+"); 677 678 private final boolean mEditTextLogger; 679 private final BreakIterator mTokenIterator; 680 681 @Nullable private TextClassifier mClassificationSession; 682 private int mStartIndex; 683 private String mText; 684 SelectionMetricsLogger(TextView textView)685 SelectionMetricsLogger(TextView textView) { 686 Preconditions.checkNotNull(textView); 687 mEditTextLogger = textView.isTextEditable(); 688 mTokenIterator = SelectionSessionLogger.getTokenIterator(textView.getTextLocale()); 689 } 690 691 @TextClassifier.WidgetType getWidetType(TextView textView)692 private static String getWidetType(TextView textView) { 693 if (textView.isTextEditable()) { 694 return TextClassifier.WIDGET_TYPE_EDITTEXT; 695 } 696 if (textView.isTextSelectable()) { 697 return TextClassifier.WIDGET_TYPE_TEXTVIEW; 698 } 699 return TextClassifier.WIDGET_TYPE_UNSELECTABLE_TEXTVIEW; 700 } 701 logSelectionStarted( TextClassifier classificationSession, CharSequence text, int index, @InvocationMethod int invocationMethod)702 public void logSelectionStarted( 703 TextClassifier classificationSession, 704 CharSequence text, int index, 705 @InvocationMethod int invocationMethod) { 706 try { 707 Preconditions.checkNotNull(text); 708 Preconditions.checkArgumentInRange(index, 0, text.length(), "index"); 709 if (mText == null || !mText.contentEquals(text)) { 710 mText = text.toString(); 711 } 712 mTokenIterator.setText(mText); 713 mStartIndex = index; 714 mClassificationSession = classificationSession; 715 if (hasActiveClassificationSession()) { 716 mClassificationSession.onSelectionEvent( 717 SelectionEvent.createSelectionStartedEvent(invocationMethod, 0)); 718 } 719 } catch (Exception e) { 720 // Avoid crashes due to logging. 721 Log.e(LOG_TAG, "" + e.getMessage(), e); 722 } 723 } 724 logSelectionModified(int start, int end, @Nullable TextClassification classification, @Nullable TextSelection selection)725 public void logSelectionModified(int start, int end, 726 @Nullable TextClassification classification, @Nullable TextSelection selection) { 727 try { 728 if (hasActiveClassificationSession()) { 729 Preconditions.checkArgumentInRange(start, 0, mText.length(), "start"); 730 Preconditions.checkArgumentInRange(end, start, mText.length(), "end"); 731 int[] wordIndices = getWordDelta(start, end); 732 if (selection != null) { 733 mClassificationSession.onSelectionEvent( 734 SelectionEvent.createSelectionModifiedEvent( 735 wordIndices[0], wordIndices[1], selection)); 736 } else if (classification != null) { 737 mClassificationSession.onSelectionEvent( 738 SelectionEvent.createSelectionModifiedEvent( 739 wordIndices[0], wordIndices[1], classification)); 740 } else { 741 mClassificationSession.onSelectionEvent( 742 SelectionEvent.createSelectionModifiedEvent( 743 wordIndices[0], wordIndices[1])); 744 } 745 } 746 } catch (Exception e) { 747 // Avoid crashes due to logging. 748 Log.e(LOG_TAG, "" + e.getMessage(), e); 749 } 750 } 751 logSelectionAction( int start, int end, @SelectionEvent.ActionType int action, @Nullable TextClassification classification)752 public void logSelectionAction( 753 int start, int end, 754 @SelectionEvent.ActionType int action, 755 @Nullable TextClassification classification) { 756 try { 757 if (hasActiveClassificationSession()) { 758 Preconditions.checkArgumentInRange(start, 0, mText.length(), "start"); 759 Preconditions.checkArgumentInRange(end, start, mText.length(), "end"); 760 int[] wordIndices = getWordDelta(start, end); 761 if (classification != null) { 762 mClassificationSession.onSelectionEvent( 763 SelectionEvent.createSelectionActionEvent( 764 wordIndices[0], wordIndices[1], action, 765 classification)); 766 } else { 767 mClassificationSession.onSelectionEvent( 768 SelectionEvent.createSelectionActionEvent( 769 wordIndices[0], wordIndices[1], action)); 770 } 771 if (SelectionEvent.isTerminal(action)) { 772 endTextClassificationSession(); 773 } 774 } 775 } catch (Exception e) { 776 // Avoid crashes due to logging. 777 Log.e(LOG_TAG, "" + e.getMessage(), e); 778 } 779 } 780 isEditTextLogger()781 public boolean isEditTextLogger() { 782 return mEditTextLogger; 783 } 784 endTextClassificationSession()785 public void endTextClassificationSession() { 786 if (hasActiveClassificationSession()) { 787 mClassificationSession.destroy(); 788 } 789 } 790 hasActiveClassificationSession()791 private boolean hasActiveClassificationSession() { 792 return mClassificationSession != null && !mClassificationSession.isDestroyed(); 793 } 794 getWordDelta(int start, int end)795 private int[] getWordDelta(int start, int end) { 796 int[] wordIndices = new int[2]; 797 798 if (start == mStartIndex) { 799 wordIndices[0] = 0; 800 } else if (start < mStartIndex) { 801 wordIndices[0] = -countWordsForward(start); 802 } else { // start > mStartIndex 803 wordIndices[0] = countWordsBackward(start); 804 805 // For the selection start index, avoid counting a partial word backwards. 806 if (!mTokenIterator.isBoundary(start) 807 && !isWhitespace( 808 mTokenIterator.preceding(start), 809 mTokenIterator.following(start))) { 810 // We counted a partial word. Remove it. 811 wordIndices[0]--; 812 } 813 } 814 815 if (end == mStartIndex) { 816 wordIndices[1] = 0; 817 } else if (end < mStartIndex) { 818 wordIndices[1] = -countWordsForward(end); 819 } else { // end > mStartIndex 820 wordIndices[1] = countWordsBackward(end); 821 } 822 823 return wordIndices; 824 } 825 countWordsBackward(int from)826 private int countWordsBackward(int from) { 827 Preconditions.checkArgument(from >= mStartIndex); 828 int wordCount = 0; 829 int offset = from; 830 while (offset > mStartIndex) { 831 int start = mTokenIterator.preceding(offset); 832 if (!isWhitespace(start, offset)) { 833 wordCount++; 834 } 835 offset = start; 836 } 837 return wordCount; 838 } 839 countWordsForward(int from)840 private int countWordsForward(int from) { 841 Preconditions.checkArgument(from <= mStartIndex); 842 int wordCount = 0; 843 int offset = from; 844 while (offset < mStartIndex) { 845 int end = mTokenIterator.following(offset); 846 if (!isWhitespace(offset, end)) { 847 wordCount++; 848 } 849 offset = end; 850 } 851 return wordCount; 852 } 853 isWhitespace(int start, int end)854 private boolean isWhitespace(int start, int end) { 855 return PATTERN_WHITESPACE.matcher(mText.substring(start, end)).matches(); 856 } 857 } 858 859 /** 860 * AsyncTask for running a query on a background thread and returning the result on the 861 * UiThread. The AsyncTask times out after a specified time, returning a null result if the 862 * query has not yet returned. 863 */ 864 private static final class TextClassificationAsyncTask 865 extends AsyncTask<Void, Void, SelectionResult> { 866 867 private final int mTimeOutDuration; 868 private final Supplier<SelectionResult> mSelectionResultSupplier; 869 private final Consumer<SelectionResult> mSelectionResultCallback; 870 private final Supplier<SelectionResult> mTimeOutResultSupplier; 871 private final TextView mTextView; 872 private final String mOriginalText; 873 874 /** 875 * @param textView the TextView 876 * @param timeOut time in milliseconds to timeout the query if it has not completed 877 * @param selectionResultSupplier fetches the selection results. Runs on a background thread 878 * @param selectionResultCallback receives the selection results. Runs on the UiThread 879 * @param timeOutResultSupplier default result if the task times out 880 */ TextClassificationAsyncTask( @onNull TextView textView, int timeOut, @NonNull Supplier<SelectionResult> selectionResultSupplier, @NonNull Consumer<SelectionResult> selectionResultCallback, @NonNull Supplier<SelectionResult> timeOutResultSupplier)881 TextClassificationAsyncTask( 882 @NonNull TextView textView, int timeOut, 883 @NonNull Supplier<SelectionResult> selectionResultSupplier, 884 @NonNull Consumer<SelectionResult> selectionResultCallback, 885 @NonNull Supplier<SelectionResult> timeOutResultSupplier) { 886 super(textView != null ? textView.getHandler() : null); 887 mTextView = Preconditions.checkNotNull(textView); 888 mTimeOutDuration = timeOut; 889 mSelectionResultSupplier = Preconditions.checkNotNull(selectionResultSupplier); 890 mSelectionResultCallback = Preconditions.checkNotNull(selectionResultCallback); 891 mTimeOutResultSupplier = Preconditions.checkNotNull(timeOutResultSupplier); 892 // Make a copy of the original text. 893 mOriginalText = getText(mTextView).toString(); 894 } 895 896 @Override 897 @WorkerThread doInBackground(Void... params)898 protected SelectionResult doInBackground(Void... params) { 899 final Runnable onTimeOut = this::onTimeOut; 900 mTextView.postDelayed(onTimeOut, mTimeOutDuration); 901 final SelectionResult result = mSelectionResultSupplier.get(); 902 mTextView.removeCallbacks(onTimeOut); 903 return result; 904 } 905 906 @Override 907 @UiThread onPostExecute(SelectionResult result)908 protected void onPostExecute(SelectionResult result) { 909 result = TextUtils.equals(mOriginalText, getText(mTextView)) ? result : null; 910 mSelectionResultCallback.accept(result); 911 } 912 onTimeOut()913 private void onTimeOut() { 914 if (getStatus() == Status.RUNNING) { 915 onPostExecute(mTimeOutResultSupplier.get()); 916 } 917 cancel(true); 918 } 919 } 920 921 /** 922 * Helper class for querying the TextClassifier. 923 * It trims text so that only text necessary to provide context of the selected text is 924 * sent to the TextClassifier. 925 */ 926 private static final class TextClassificationHelper { 927 928 private static final int TRIM_DELTA = 120; // characters 929 930 private final Context mContext; 931 private Supplier<TextClassifier> mTextClassifier; 932 933 /** The original TextView text. **/ 934 private String mText; 935 /** Start index relative to mText. */ 936 private int mSelectionStart; 937 /** End index relative to mText. */ 938 private int mSelectionEnd; 939 940 @Nullable 941 private LocaleList mDefaultLocales; 942 943 /** Trimmed text starting from mTrimStart in mText. */ 944 private CharSequence mTrimmedText; 945 /** Index indicating the start of mTrimmedText in mText. */ 946 private int mTrimStart; 947 /** Start index relative to mTrimmedText */ 948 private int mRelativeStart; 949 /** End index relative to mTrimmedText */ 950 private int mRelativeEnd; 951 952 /** Information about the last classified text to avoid re-running a query. */ 953 private CharSequence mLastClassificationText; 954 private int mLastClassificationSelectionStart; 955 private int mLastClassificationSelectionEnd; 956 private LocaleList mLastClassificationLocales; 957 private SelectionResult mLastClassificationResult; 958 959 /** Whether the TextClassifier has been initialized. */ 960 private boolean mHot; 961 TextClassificationHelper(Context context, Supplier<TextClassifier> textClassifier, CharSequence text, int selectionStart, int selectionEnd, LocaleList locales)962 TextClassificationHelper(Context context, Supplier<TextClassifier> textClassifier, 963 CharSequence text, int selectionStart, int selectionEnd, LocaleList locales) { 964 init(textClassifier, text, selectionStart, selectionEnd, locales); 965 mContext = Preconditions.checkNotNull(context); 966 } 967 968 @UiThread init(Supplier<TextClassifier> textClassifier, CharSequence text, int selectionStart, int selectionEnd, LocaleList locales)969 public void init(Supplier<TextClassifier> textClassifier, CharSequence text, 970 int selectionStart, int selectionEnd, LocaleList locales) { 971 mTextClassifier = Preconditions.checkNotNull(textClassifier); 972 mText = Preconditions.checkNotNull(text).toString(); 973 mLastClassificationText = null; // invalidate. 974 Preconditions.checkArgument(selectionEnd > selectionStart); 975 mSelectionStart = selectionStart; 976 mSelectionEnd = selectionEnd; 977 mDefaultLocales = locales; 978 } 979 980 @WorkerThread classifyText()981 public SelectionResult classifyText() { 982 mHot = true; 983 return performClassification(null /* selection */); 984 } 985 986 @WorkerThread suggestSelection()987 public SelectionResult suggestSelection() { 988 mHot = true; 989 trimText(); 990 final TextSelection selection; 991 if (mContext.getApplicationInfo().targetSdkVersion >= Build.VERSION_CODES.P) { 992 final TextSelection.Request request = new TextSelection.Request.Builder( 993 mTrimmedText, mRelativeStart, mRelativeEnd) 994 .setDefaultLocales(mDefaultLocales) 995 .setDarkLaunchAllowed(true) 996 .build(); 997 selection = mTextClassifier.get().suggestSelection(request); 998 } else { 999 // Use old APIs. 1000 selection = mTextClassifier.get().suggestSelection( 1001 mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales); 1002 } 1003 // Do not classify new selection boundaries if TextClassifier should be dark launched. 1004 if (!isDarkLaunchEnabled()) { 1005 mSelectionStart = Math.max(0, selection.getSelectionStartIndex() + mTrimStart); 1006 mSelectionEnd = Math.min( 1007 mText.length(), selection.getSelectionEndIndex() + mTrimStart); 1008 } 1009 return performClassification(selection); 1010 } 1011 getOriginalSelection()1012 public SelectionResult getOriginalSelection() { 1013 return new SelectionResult(mSelectionStart, mSelectionEnd, null, null); 1014 } 1015 1016 /** 1017 * Maximum time (in milliseconds) to wait for a textclassifier result before timing out. 1018 */ 1019 // TODO: Consider making this a ViewConfiguration. getTimeoutDuration()1020 public int getTimeoutDuration() { 1021 if (mHot) { 1022 return 200; 1023 } else { 1024 // Return a slightly larger number than usual when the TextClassifier is first 1025 // initialized. Initialization would usually take longer than subsequent calls to 1026 // the TextClassifier. The impact of this on the UI is that we do not show the 1027 // selection handles or toolbar until after this timeout. 1028 return 500; 1029 } 1030 } 1031 isDarkLaunchEnabled()1032 private boolean isDarkLaunchEnabled() { 1033 return TextClassificationManager.getSettings(mContext).isModelDarkLaunchEnabled(); 1034 } 1035 performClassification(@ullable TextSelection selection)1036 private SelectionResult performClassification(@Nullable TextSelection selection) { 1037 if (!Objects.equals(mText, mLastClassificationText) 1038 || mSelectionStart != mLastClassificationSelectionStart 1039 || mSelectionEnd != mLastClassificationSelectionEnd 1040 || !Objects.equals(mDefaultLocales, mLastClassificationLocales)) { 1041 1042 mLastClassificationText = mText; 1043 mLastClassificationSelectionStart = mSelectionStart; 1044 mLastClassificationSelectionEnd = mSelectionEnd; 1045 mLastClassificationLocales = mDefaultLocales; 1046 1047 trimText(); 1048 final TextClassification classification; 1049 if (Linkify.containsUnsupportedCharacters(mText)) { 1050 // Do not show smart actions for text containing unsupported characters. 1051 android.util.EventLog.writeEvent(0x534e4554, "116321860", -1, ""); 1052 classification = TextClassification.EMPTY; 1053 } else if (mContext.getApplicationInfo().targetSdkVersion 1054 >= Build.VERSION_CODES.P) { 1055 final TextClassification.Request request = 1056 new TextClassification.Request.Builder( 1057 mTrimmedText, mRelativeStart, mRelativeEnd) 1058 .setDefaultLocales(mDefaultLocales) 1059 .build(); 1060 classification = mTextClassifier.get().classifyText(request); 1061 } else { 1062 // Use old APIs. 1063 classification = mTextClassifier.get().classifyText( 1064 mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales); 1065 } 1066 mLastClassificationResult = new SelectionResult( 1067 mSelectionStart, mSelectionEnd, classification, selection); 1068 1069 } 1070 return mLastClassificationResult; 1071 } 1072 trimText()1073 private void trimText() { 1074 mTrimStart = Math.max(0, mSelectionStart - TRIM_DELTA); 1075 final int referenceEnd = Math.min(mText.length(), mSelectionEnd + TRIM_DELTA); 1076 mTrimmedText = mText.subSequence(mTrimStart, referenceEnd); 1077 mRelativeStart = mSelectionStart - mTrimStart; 1078 mRelativeEnd = mSelectionEnd - mTrimStart; 1079 } 1080 } 1081 1082 /** 1083 * Selection result. 1084 */ 1085 private static final class SelectionResult { 1086 private final int mStart; 1087 private final int mEnd; 1088 @Nullable private final TextClassification mClassification; 1089 @Nullable private final TextSelection mSelection; 1090 SelectionResult(int start, int end, @Nullable TextClassification classification, @Nullable TextSelection selection)1091 SelectionResult(int start, int end, 1092 @Nullable TextClassification classification, @Nullable TextSelection selection) { 1093 mStart = start; 1094 mEnd = end; 1095 mClassification = classification; 1096 mSelection = selection; 1097 } 1098 } 1099 1100 @SelectionEvent.ActionType getActionType(int menuItemId)1101 private static int getActionType(int menuItemId) { 1102 switch (menuItemId) { 1103 case TextView.ID_SELECT_ALL: 1104 return SelectionEvent.ACTION_SELECT_ALL; 1105 case TextView.ID_CUT: 1106 return SelectionEvent.ACTION_CUT; 1107 case TextView.ID_COPY: 1108 return SelectionEvent.ACTION_COPY; 1109 case TextView.ID_PASTE: // fall through 1110 case TextView.ID_PASTE_AS_PLAIN_TEXT: 1111 return SelectionEvent.ACTION_PASTE; 1112 case TextView.ID_SHARE: 1113 return SelectionEvent.ACTION_SHARE; 1114 case TextView.ID_ASSIST: 1115 return SelectionEvent.ACTION_SMART_SHARE; 1116 default: 1117 return SelectionEvent.ACTION_OTHER; 1118 } 1119 } 1120 getText(TextView textView)1121 private static CharSequence getText(TextView textView) { 1122 // Extracts the textView's text. 1123 // TODO: Investigate why/when TextView.getText() is null. 1124 final CharSequence text = textView.getText(); 1125 if (text != null) { 1126 return text; 1127 } 1128 return ""; 1129 } 1130 } 1131