• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.widget;
18 
19 import android.annotation.NonNull;
20 import android.annotation.Nullable;
21 import android.annotation.UiThread;
22 import android.annotation.WorkerThread;
23 import android.content.Context;
24 import android.graphics.Canvas;
25 import android.graphics.PointF;
26 import android.graphics.RectF;
27 import android.os.AsyncTask;
28 import android.os.Build;
29 import android.os.LocaleList;
30 import android.text.Layout;
31 import android.text.Selection;
32 import android.text.Spannable;
33 import android.text.TextUtils;
34 import android.text.util.Linkify;
35 import android.util.Log;
36 import android.view.ActionMode;
37 import android.view.textclassifier.SelectionEvent;
38 import android.view.textclassifier.SelectionEvent.InvocationMethod;
39 import android.view.textclassifier.SelectionSessionLogger;
40 import android.view.textclassifier.TextClassification;
41 import android.view.textclassifier.TextClassificationConstants;
42 import android.view.textclassifier.TextClassificationManager;
43 import android.view.textclassifier.TextClassifier;
44 import android.view.textclassifier.TextSelection;
45 import android.widget.Editor.SelectionModifierCursorController;
46 
47 import com.android.internal.annotations.VisibleForTesting;
48 import com.android.internal.util.Preconditions;
49 
50 import java.text.BreakIterator;
51 import java.util.ArrayList;
52 import java.util.Comparator;
53 import java.util.List;
54 import java.util.Objects;
55 import java.util.function.Consumer;
56 import java.util.function.Function;
57 import java.util.function.Supplier;
58 import java.util.regex.Pattern;
59 
60 /**
61  * Helper class for starting selection action mode
62  * (synchronously without the TextClassifier, asynchronously with the TextClassifier).
63  * @hide
64  */
65 @UiThread
66 @VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE)
67 public final class SelectionActionModeHelper {
68 
69     private static final String LOG_TAG = "SelectActionModeHelper";
70 
71     private final Editor mEditor;
72     private final TextView mTextView;
73     private final TextClassificationHelper mTextClassificationHelper;
74 
75     @Nullable private TextClassification mTextClassification;
76     private AsyncTask mTextClassificationAsyncTask;
77 
78     private final SelectionTracker mSelectionTracker;
79 
80     // TODO remove nullable marker once the switch gating the feature gets removed
81     @Nullable
82     private final SmartSelectSprite mSmartSelectSprite;
83 
SelectionActionModeHelper(@onNull Editor editor)84     SelectionActionModeHelper(@NonNull Editor editor) {
85         mEditor = Preconditions.checkNotNull(editor);
86         mTextView = mEditor.getTextView();
87         mTextClassificationHelper = new TextClassificationHelper(
88                 mTextView.getContext(),
89                 mTextView::getTextClassifier,
90                 getText(mTextView),
91                 0, 1, mTextView.getTextLocales());
92         mSelectionTracker = new SelectionTracker(mTextView);
93 
94         if (getTextClassificationSettings().isSmartSelectionAnimationEnabled()) {
95             mSmartSelectSprite = new SmartSelectSprite(mTextView.getContext(),
96                     editor.getTextView().mHighlightColor, mTextView::invalidate);
97         } else {
98             mSmartSelectSprite = null;
99         }
100     }
101 
102     /**
103      * Starts Selection ActionMode.
104      */
startSelectionActionModeAsync(boolean adjustSelection)105     public void startSelectionActionModeAsync(boolean adjustSelection) {
106         // Check if the smart selection should run for editable text.
107         adjustSelection &= getTextClassificationSettings().isSmartSelectionEnabled();
108 
109         mSelectionTracker.onOriginalSelection(
110                 getText(mTextView),
111                 mTextView.getSelectionStart(),
112                 mTextView.getSelectionEnd(),
113                 false /*isLink*/);
114         cancelAsyncTask();
115         if (skipTextClassification()) {
116             startSelectionActionMode(null);
117         } else {
118             resetTextClassificationHelper();
119             mTextClassificationAsyncTask = new TextClassificationAsyncTask(
120                     mTextView,
121                     mTextClassificationHelper.getTimeoutDuration(),
122                     adjustSelection
123                             ? mTextClassificationHelper::suggestSelection
124                             : mTextClassificationHelper::classifyText,
125                     mSmartSelectSprite != null
126                             ? this::startSelectionActionModeWithSmartSelectAnimation
127                             : this::startSelectionActionMode,
128                     mTextClassificationHelper::getOriginalSelection)
129                     .execute();
130         }
131     }
132 
133     /**
134      * Starts Link ActionMode.
135      */
startLinkActionModeAsync(int start, int end)136     public void startLinkActionModeAsync(int start, int end) {
137         mSelectionTracker.onOriginalSelection(getText(mTextView), start, end, true /*isLink*/);
138         cancelAsyncTask();
139         if (skipTextClassification()) {
140             startLinkActionMode(null);
141         } else {
142             resetTextClassificationHelper(start, end);
143             mTextClassificationAsyncTask = new TextClassificationAsyncTask(
144                     mTextView,
145                     mTextClassificationHelper.getTimeoutDuration(),
146                     mTextClassificationHelper::classifyText,
147                     this::startLinkActionMode,
148                     mTextClassificationHelper::getOriginalSelection)
149                     .execute();
150         }
151     }
152 
invalidateActionModeAsync()153     public void invalidateActionModeAsync() {
154         cancelAsyncTask();
155         if (skipTextClassification()) {
156             invalidateActionMode(null);
157         } else {
158             resetTextClassificationHelper();
159             mTextClassificationAsyncTask = new TextClassificationAsyncTask(
160                     mTextView,
161                     mTextClassificationHelper.getTimeoutDuration(),
162                     mTextClassificationHelper::classifyText,
163                     this::invalidateActionMode,
164                     mTextClassificationHelper::getOriginalSelection)
165                     .execute();
166         }
167     }
168 
onSelectionAction(int menuItemId)169     public void onSelectionAction(int menuItemId) {
170         mSelectionTracker.onSelectionAction(
171                 mTextView.getSelectionStart(), mTextView.getSelectionEnd(),
172                 getActionType(menuItemId), mTextClassification);
173     }
174 
onSelectionDrag()175     public void onSelectionDrag() {
176         mSelectionTracker.onSelectionAction(
177                 mTextView.getSelectionStart(), mTextView.getSelectionEnd(),
178                 SelectionEvent.ACTION_DRAG, mTextClassification);
179     }
180 
onTextChanged(int start, int end)181     public void onTextChanged(int start, int end) {
182         mSelectionTracker.onTextChanged(start, end, mTextClassification);
183     }
184 
resetSelection(int textIndex)185     public boolean resetSelection(int textIndex) {
186         if (mSelectionTracker.resetSelection(textIndex, mEditor)) {
187             invalidateActionModeAsync();
188             return true;
189         }
190         return false;
191     }
192 
193     @Nullable
getTextClassification()194     public TextClassification getTextClassification() {
195         return mTextClassification;
196     }
197 
onDestroyActionMode()198     public void onDestroyActionMode() {
199         cancelSmartSelectAnimation();
200         mSelectionTracker.onSelectionDestroyed();
201         cancelAsyncTask();
202     }
203 
onDraw(final Canvas canvas)204     public void onDraw(final Canvas canvas) {
205         if (isDrawingHighlight() && mSmartSelectSprite != null) {
206             mSmartSelectSprite.draw(canvas);
207         }
208     }
209 
isDrawingHighlight()210     public boolean isDrawingHighlight() {
211         return mSmartSelectSprite != null && mSmartSelectSprite.isAnimationActive();
212     }
213 
getTextClassificationSettings()214     private TextClassificationConstants getTextClassificationSettings() {
215         return TextClassificationManager.getSettings(mTextView.getContext());
216     }
217 
cancelAsyncTask()218     private void cancelAsyncTask() {
219         if (mTextClassificationAsyncTask != null) {
220             mTextClassificationAsyncTask.cancel(true);
221             mTextClassificationAsyncTask = null;
222         }
223         mTextClassification = null;
224     }
225 
skipTextClassification()226     private boolean skipTextClassification() {
227         // No need to make an async call for a no-op TextClassifier.
228         final boolean noOpTextClassifier = mTextView.usesNoOpTextClassifier();
229         // Do not call the TextClassifier if there is no selection.
230         final boolean noSelection = mTextView.getSelectionEnd() == mTextView.getSelectionStart();
231         // Do not call the TextClassifier if this is a password field.
232         final boolean password = mTextView.hasPasswordTransformationMethod()
233                 || TextView.isPasswordInputType(mTextView.getInputType());
234         return noOpTextClassifier || noSelection || password;
235     }
236 
startLinkActionMode(@ullable SelectionResult result)237     private void startLinkActionMode(@Nullable SelectionResult result) {
238         startActionMode(Editor.TextActionMode.TEXT_LINK, result);
239     }
240 
startSelectionActionMode(@ullable SelectionResult result)241     private void startSelectionActionMode(@Nullable SelectionResult result) {
242         startActionMode(Editor.TextActionMode.SELECTION, result);
243     }
244 
startActionMode( @ditor.TextActionMode int actionMode, @Nullable SelectionResult result)245     private void startActionMode(
246             @Editor.TextActionMode int actionMode, @Nullable SelectionResult result) {
247         final CharSequence text = getText(mTextView);
248         if (result != null && text instanceof Spannable
249                 && (mTextView.isTextSelectable() || mTextView.isTextEditable())) {
250             // Do not change the selection if TextClassifier should be dark launched.
251             if (!getTextClassificationSettings().isModelDarkLaunchEnabled()) {
252                 Selection.setSelection((Spannable) text, result.mStart, result.mEnd);
253                 mTextView.invalidate();
254             }
255             mTextClassification = result.mClassification;
256         } else if (result != null && actionMode == Editor.TextActionMode.TEXT_LINK) {
257             mTextClassification = result.mClassification;
258         } else {
259             mTextClassification = null;
260         }
261         if (mEditor.startActionModeInternal(actionMode)) {
262             final SelectionModifierCursorController controller = mEditor.getSelectionController();
263             if (controller != null
264                     && (mTextView.isTextSelectable() || mTextView.isTextEditable())) {
265                 controller.show();
266             }
267             if (result != null) {
268                 switch (actionMode) {
269                     case Editor.TextActionMode.SELECTION:
270                         mSelectionTracker.onSmartSelection(result);
271                         break;
272                     case Editor.TextActionMode.TEXT_LINK:
273                         mSelectionTracker.onLinkSelected(result);
274                         break;
275                     default:
276                         break;
277                 }
278             }
279         }
280         mEditor.setRestartActionModeOnNextRefresh(false);
281         mTextClassificationAsyncTask = null;
282     }
283 
startSelectionActionModeWithSmartSelectAnimation( @ullable SelectionResult result)284     private void startSelectionActionModeWithSmartSelectAnimation(
285             @Nullable SelectionResult result) {
286         final Layout layout = mTextView.getLayout();
287 
288         final Runnable onAnimationEndCallback = () -> {
289             final SelectionResult startSelectionResult;
290             if (result != null && result.mStart >= 0 && result.mEnd <= getText(mTextView).length()
291                     && result.mStart <= result.mEnd) {
292                 startSelectionResult = result;
293             } else {
294                 startSelectionResult = null;
295             }
296             startSelectionActionMode(startSelectionResult);
297         };
298         // TODO do not trigger the animation if the change included only non-printable characters
299         final boolean didSelectionChange =
300                 result != null && (mTextView.getSelectionStart() != result.mStart
301                         || mTextView.getSelectionEnd() != result.mEnd);
302 
303         if (!didSelectionChange) {
304             onAnimationEndCallback.run();
305             return;
306         }
307 
308         final List<SmartSelectSprite.RectangleWithTextSelectionLayout> selectionRectangles =
309                 convertSelectionToRectangles(layout, result.mStart, result.mEnd);
310 
311         final PointF touchPoint = new PointF(
312                 mEditor.getLastUpPositionX(),
313                 mEditor.getLastUpPositionY());
314 
315         final PointF animationStartPoint =
316                 movePointInsideNearestRectangle(touchPoint, selectionRectangles,
317                         SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle);
318 
319         mSmartSelectSprite.startAnimation(
320                 animationStartPoint,
321                 selectionRectangles,
322                 onAnimationEndCallback);
323     }
324 
convertSelectionToRectangles( final Layout layout, final int start, final int end)325     private List<SmartSelectSprite.RectangleWithTextSelectionLayout> convertSelectionToRectangles(
326             final Layout layout, final int start, final int end) {
327         final List<SmartSelectSprite.RectangleWithTextSelectionLayout> result = new ArrayList<>();
328 
329         final Layout.SelectionRectangleConsumer consumer =
330                 (left, top, right, bottom, textSelectionLayout) -> mergeRectangleIntoList(
331                         result,
332                         new RectF(left, top, right, bottom),
333                         SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle,
334                         r -> new SmartSelectSprite.RectangleWithTextSelectionLayout(r,
335                                 textSelectionLayout)
336                 );
337 
338         layout.getSelection(start, end, consumer);
339 
340         result.sort(Comparator.comparing(
341                 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle,
342                 SmartSelectSprite.RECTANGLE_COMPARATOR));
343 
344         return result;
345     }
346 
347     // TODO: Move public pure functions out of this class and make it package-private.
348     /**
349      * Merges a {@link RectF} into an existing list of any objects which contain a rectangle.
350      * While merging, this method makes sure that:
351      *
352      * <ol>
353      * <li>No rectangle is redundant (contained within a bigger rectangle)</li>
354      * <li>Rectangles of the same height and vertical position that intersect get merged</li>
355      * </ol>
356      *
357      * @param list      the list of rectangles (or other rectangle containers) to merge the new
358      *                  rectangle into
359      * @param candidate the {@link RectF} to merge into the list
360      * @param extractor a function that can extract a {@link RectF} from an element of the given
361      *                  list
362      * @param packer    a function that can wrap the resulting {@link RectF} into an element that
363      *                  the list contains
364      * @hide
365      */
366     @VisibleForTesting
mergeRectangleIntoList(final List<T> list, final RectF candidate, final Function<T, RectF> extractor, final Function<RectF, T> packer)367     public static <T> void mergeRectangleIntoList(final List<T> list,
368             final RectF candidate, final Function<T, RectF> extractor,
369             final Function<RectF, T> packer) {
370         if (candidate.isEmpty()) {
371             return;
372         }
373 
374         final int elementCount = list.size();
375         for (int index = 0; index < elementCount; ++index) {
376             final RectF existingRectangle = extractor.apply(list.get(index));
377             if (existingRectangle.contains(candidate)) {
378                 return;
379             }
380             if (candidate.contains(existingRectangle)) {
381                 existingRectangle.setEmpty();
382                 continue;
383             }
384 
385             final boolean rectanglesContinueEachOther = candidate.left == existingRectangle.right
386                     || candidate.right == existingRectangle.left;
387             final boolean canMerge = candidate.top == existingRectangle.top
388                     && candidate.bottom == existingRectangle.bottom
389                     && (RectF.intersects(candidate, existingRectangle)
390                     || rectanglesContinueEachOther);
391 
392             if (canMerge) {
393                 candidate.union(existingRectangle);
394                 existingRectangle.setEmpty();
395             }
396         }
397 
398         for (int index = elementCount - 1; index >= 0; --index) {
399             final RectF rectangle = extractor.apply(list.get(index));
400             if (rectangle.isEmpty()) {
401                 list.remove(index);
402             }
403         }
404 
405         list.add(packer.apply(candidate));
406     }
407 
408 
409     /** @hide */
410     @VisibleForTesting
movePointInsideNearestRectangle(final PointF point, final List<T> list, final Function<T, RectF> extractor)411     public static <T> PointF movePointInsideNearestRectangle(final PointF point,
412             final List<T> list, final Function<T, RectF> extractor) {
413         float bestX = -1;
414         float bestY = -1;
415         double bestDistance = Double.MAX_VALUE;
416 
417         final int elementCount = list.size();
418         for (int index = 0; index < elementCount; ++index) {
419             final RectF rectangle = extractor.apply(list.get(index));
420             final float candidateY = rectangle.centerY();
421             final float candidateX;
422 
423             if (point.x > rectangle.right) {
424                 candidateX = rectangle.right;
425             } else if (point.x < rectangle.left) {
426                 candidateX = rectangle.left;
427             } else {
428                 candidateX = point.x;
429             }
430 
431             final double candidateDistance = Math.pow(point.x - candidateX, 2)
432                     + Math.pow(point.y - candidateY, 2);
433 
434             if (candidateDistance < bestDistance) {
435                 bestX = candidateX;
436                 bestY = candidateY;
437                 bestDistance = candidateDistance;
438             }
439         }
440 
441         return new PointF(bestX, bestY);
442     }
443 
invalidateActionMode(@ullable SelectionResult result)444     private void invalidateActionMode(@Nullable SelectionResult result) {
445         cancelSmartSelectAnimation();
446         mTextClassification = result != null ? result.mClassification : null;
447         final ActionMode actionMode = mEditor.getTextActionMode();
448         if (actionMode != null) {
449             actionMode.invalidate();
450         }
451         mSelectionTracker.onSelectionUpdated(
452                 mTextView.getSelectionStart(), mTextView.getSelectionEnd(), mTextClassification);
453         mTextClassificationAsyncTask = null;
454     }
455 
resetTextClassificationHelper(int selectionStart, int selectionEnd)456     private void resetTextClassificationHelper(int selectionStart, int selectionEnd) {
457         if (selectionStart < 0 || selectionEnd < 0) {
458             // Use selection indices
459             selectionStart = mTextView.getSelectionStart();
460             selectionEnd = mTextView.getSelectionEnd();
461         }
462         mTextClassificationHelper.init(
463                 mTextView::getTextClassifier,
464                 getText(mTextView),
465                 selectionStart, selectionEnd,
466                 mTextView.getTextLocales());
467     }
468 
resetTextClassificationHelper()469     private void resetTextClassificationHelper() {
470         resetTextClassificationHelper(-1, -1);
471     }
472 
cancelSmartSelectAnimation()473     private void cancelSmartSelectAnimation() {
474         if (mSmartSelectSprite != null) {
475             mSmartSelectSprite.cancelAnimation();
476         }
477     }
478 
479     /**
480      * Tracks and logs smart selection changes.
481      * It is important to trigger this object's methods at the appropriate event so that it tracks
482      * smart selection events appropriately.
483      */
484     private static final class SelectionTracker {
485 
486         private final TextView mTextView;
487         private SelectionMetricsLogger mLogger;
488 
489         private int mOriginalStart;
490         private int mOriginalEnd;
491         private int mSelectionStart;
492         private int mSelectionEnd;
493         private boolean mAllowReset;
494         private final LogAbandonRunnable mDelayedLogAbandon = new LogAbandonRunnable();
495 
SelectionTracker(TextView textView)496         SelectionTracker(TextView textView) {
497             mTextView = Preconditions.checkNotNull(textView);
498             mLogger = new SelectionMetricsLogger(textView);
499         }
500 
501         /**
502          * Called when the original selection happens, before smart selection is triggered.
503          */
onOriginalSelection( CharSequence text, int selectionStart, int selectionEnd, boolean isLink)504         public void onOriginalSelection(
505                 CharSequence text, int selectionStart, int selectionEnd, boolean isLink) {
506             // If we abandoned a selection and created a new one very shortly after, we may still
507             // have a pending request to log ABANDON, which we flush here.
508             mDelayedLogAbandon.flush();
509 
510             mOriginalStart = mSelectionStart = selectionStart;
511             mOriginalEnd = mSelectionEnd = selectionEnd;
512             mAllowReset = false;
513             maybeInvalidateLogger();
514             mLogger.logSelectionStarted(mTextView.getTextClassificationSession(),
515                     text, selectionStart,
516                     isLink ? SelectionEvent.INVOCATION_LINK : SelectionEvent.INVOCATION_MANUAL);
517         }
518 
519         /**
520          * Called when selection action mode is started and the results come from a classifier.
521          */
onSmartSelection(SelectionResult result)522         public void onSmartSelection(SelectionResult result) {
523             onClassifiedSelection(result);
524             mLogger.logSelectionModified(
525                     result.mStart, result.mEnd, result.mClassification, result.mSelection);
526         }
527 
528         /**
529          * Called when link action mode is started and the classification comes from a classifier.
530          */
onLinkSelected(SelectionResult result)531         public void onLinkSelected(SelectionResult result) {
532             onClassifiedSelection(result);
533             // TODO: log (b/70246800)
534         }
535 
onClassifiedSelection(SelectionResult result)536         private void onClassifiedSelection(SelectionResult result) {
537             if (isSelectionStarted()) {
538                 mSelectionStart = result.mStart;
539                 mSelectionEnd = result.mEnd;
540                 mAllowReset = mSelectionStart != mOriginalStart || mSelectionEnd != mOriginalEnd;
541             }
542         }
543 
544         /**
545          * Called when selection bounds change.
546          */
onSelectionUpdated( int selectionStart, int selectionEnd, @Nullable TextClassification classification)547         public void onSelectionUpdated(
548                 int selectionStart, int selectionEnd,
549                 @Nullable TextClassification classification) {
550             if (isSelectionStarted()) {
551                 mSelectionStart = selectionStart;
552                 mSelectionEnd = selectionEnd;
553                 mAllowReset = false;
554                 mLogger.logSelectionModified(selectionStart, selectionEnd, classification, null);
555             }
556         }
557 
558         /**
559          * Called when the selection action mode is destroyed.
560          */
onSelectionDestroyed()561         public void onSelectionDestroyed() {
562             mAllowReset = false;
563             // Wait a few ms to see if the selection was destroyed because of a text change event.
564             mDelayedLogAbandon.schedule(100 /* ms */);
565         }
566 
567         /**
568          * Called when an action is taken on a smart selection.
569          */
onSelectionAction( int selectionStart, int selectionEnd, @SelectionEvent.ActionType int action, @Nullable TextClassification classification)570         public void onSelectionAction(
571                 int selectionStart, int selectionEnd,
572                 @SelectionEvent.ActionType int action,
573                 @Nullable TextClassification classification) {
574             if (isSelectionStarted()) {
575                 mAllowReset = false;
576                 mLogger.logSelectionAction(selectionStart, selectionEnd, action, classification);
577             }
578         }
579 
580         /**
581          * Returns true if the current smart selection should be reset to normal selection based on
582          * information that has been recorded about the original selection and the smart selection.
583          * The expected UX here is to allow the user to select a word inside of the smart selection
584          * on a single tap.
585          */
resetSelection(int textIndex, Editor editor)586         public boolean resetSelection(int textIndex, Editor editor) {
587             final TextView textView = editor.getTextView();
588             if (isSelectionStarted()
589                     && mAllowReset
590                     && textIndex >= mSelectionStart && textIndex <= mSelectionEnd
591                     && getText(textView) instanceof Spannable) {
592                 mAllowReset = false;
593                 boolean selected = editor.selectCurrentWord();
594                 if (selected) {
595                     mSelectionStart = editor.getTextView().getSelectionStart();
596                     mSelectionEnd = editor.getTextView().getSelectionEnd();
597                     mLogger.logSelectionAction(
598                             textView.getSelectionStart(), textView.getSelectionEnd(),
599                             SelectionEvent.ACTION_RESET, null /* classification */);
600                 }
601                 return selected;
602             }
603             return false;
604         }
605 
onTextChanged(int start, int end, TextClassification classification)606         public void onTextChanged(int start, int end, TextClassification classification) {
607             if (isSelectionStarted() && start == mSelectionStart && end == mSelectionEnd) {
608                 onSelectionAction(start, end, SelectionEvent.ACTION_OVERTYPE, classification);
609             }
610         }
611 
maybeInvalidateLogger()612         private void maybeInvalidateLogger() {
613             if (mLogger.isEditTextLogger() != mTextView.isTextEditable()) {
614                 mLogger = new SelectionMetricsLogger(mTextView);
615             }
616         }
617 
isSelectionStarted()618         private boolean isSelectionStarted() {
619             return mSelectionStart >= 0 && mSelectionEnd >= 0 && mSelectionStart != mSelectionEnd;
620         }
621 
622         /** A helper for keeping track of pending abandon logging requests. */
623         private final class LogAbandonRunnable implements Runnable {
624             private boolean mIsPending;
625 
626             /** Schedules an abandon to be logged with the given delay. Flush if necessary. */
schedule(int delayMillis)627             void schedule(int delayMillis) {
628                 if (mIsPending) {
629                     Log.e(LOG_TAG, "Force flushing abandon due to new scheduling request");
630                     flush();
631                 }
632                 mIsPending = true;
633                 mTextView.postDelayed(this, delayMillis);
634             }
635 
636             /** If there is a pending log request, execute it now. */
flush()637             void flush() {
638                 mTextView.removeCallbacks(this);
639                 run();
640             }
641 
642             @Override
run()643             public void run() {
644                 if (mIsPending) {
645                     mLogger.logSelectionAction(
646                             mSelectionStart, mSelectionEnd,
647                             SelectionEvent.ACTION_ABANDON, null /* classification */);
648                     mSelectionStart = mSelectionEnd = -1;
649                     mLogger.endTextClassificationSession();
650                     mIsPending = false;
651                 }
652             }
653         }
654     }
655 
656     // TODO: Write tests
657     /**
658      * Metrics logging helper.
659      *
660      * This logger logs selection by word indices. The initial (start) single word selection is
661      * logged at [0, 1) -- end index is exclusive. Other word indices are logged relative to the
662      * initial single word selection.
663      * e.g. New York city, NY. Suppose the initial selection is "York" in
664      * "New York city, NY", then "York" is at [0, 1), "New" is at [-1, 0], and "city" is at [1, 2).
665      * "New York" is at [-1, 1).
666      * Part selection of a word e.g. "or" is counted as selecting the
667      * entire word i.e. equivalent to "York", and each special character is counted as a word, e.g.
668      * "," is at [2, 3). Whitespaces are ignored.
669      *
670      * NOTE that the definition of a word is defined by the TextClassifier's Logger's token
671      * iterator.
672      */
673     private static final class SelectionMetricsLogger {
674 
675         private static final String LOG_TAG = "SelectionMetricsLogger";
676         private static final Pattern PATTERN_WHITESPACE = Pattern.compile("\\s+");
677 
678         private final boolean mEditTextLogger;
679         private final BreakIterator mTokenIterator;
680 
681         @Nullable private TextClassifier mClassificationSession;
682         private int mStartIndex;
683         private String mText;
684 
SelectionMetricsLogger(TextView textView)685         SelectionMetricsLogger(TextView textView) {
686             Preconditions.checkNotNull(textView);
687             mEditTextLogger = textView.isTextEditable();
688             mTokenIterator = SelectionSessionLogger.getTokenIterator(textView.getTextLocale());
689         }
690 
691         @TextClassifier.WidgetType
getWidetType(TextView textView)692         private static String getWidetType(TextView textView) {
693             if (textView.isTextEditable()) {
694                 return TextClassifier.WIDGET_TYPE_EDITTEXT;
695             }
696             if (textView.isTextSelectable()) {
697                 return TextClassifier.WIDGET_TYPE_TEXTVIEW;
698             }
699             return TextClassifier.WIDGET_TYPE_UNSELECTABLE_TEXTVIEW;
700         }
701 
logSelectionStarted( TextClassifier classificationSession, CharSequence text, int index, @InvocationMethod int invocationMethod)702         public void logSelectionStarted(
703                 TextClassifier classificationSession,
704                 CharSequence text, int index,
705                 @InvocationMethod int invocationMethod) {
706             try {
707                 Preconditions.checkNotNull(text);
708                 Preconditions.checkArgumentInRange(index, 0, text.length(), "index");
709                 if (mText == null || !mText.contentEquals(text)) {
710                     mText = text.toString();
711                 }
712                 mTokenIterator.setText(mText);
713                 mStartIndex = index;
714                 mClassificationSession = classificationSession;
715                 if (hasActiveClassificationSession()) {
716                     mClassificationSession.onSelectionEvent(
717                             SelectionEvent.createSelectionStartedEvent(invocationMethod, 0));
718                 }
719             } catch (Exception e) {
720                 // Avoid crashes due to logging.
721                 Log.e(LOG_TAG, "" + e.getMessage(), e);
722             }
723         }
724 
logSelectionModified(int start, int end, @Nullable TextClassification classification, @Nullable TextSelection selection)725         public void logSelectionModified(int start, int end,
726                 @Nullable TextClassification classification, @Nullable TextSelection selection) {
727             try {
728                 if (hasActiveClassificationSession()) {
729                     Preconditions.checkArgumentInRange(start, 0, mText.length(), "start");
730                     Preconditions.checkArgumentInRange(end, start, mText.length(), "end");
731                     int[] wordIndices = getWordDelta(start, end);
732                     if (selection != null) {
733                         mClassificationSession.onSelectionEvent(
734                                 SelectionEvent.createSelectionModifiedEvent(
735                                         wordIndices[0], wordIndices[1], selection));
736                     } else if (classification != null) {
737                         mClassificationSession.onSelectionEvent(
738                                 SelectionEvent.createSelectionModifiedEvent(
739                                         wordIndices[0], wordIndices[1], classification));
740                     } else {
741                         mClassificationSession.onSelectionEvent(
742                                 SelectionEvent.createSelectionModifiedEvent(
743                                         wordIndices[0], wordIndices[1]));
744                     }
745                 }
746             } catch (Exception e) {
747                 // Avoid crashes due to logging.
748                 Log.e(LOG_TAG, "" + e.getMessage(), e);
749             }
750         }
751 
logSelectionAction( int start, int end, @SelectionEvent.ActionType int action, @Nullable TextClassification classification)752         public void logSelectionAction(
753                 int start, int end,
754                 @SelectionEvent.ActionType int action,
755                 @Nullable TextClassification classification) {
756             try {
757                 if (hasActiveClassificationSession()) {
758                     Preconditions.checkArgumentInRange(start, 0, mText.length(), "start");
759                     Preconditions.checkArgumentInRange(end, start, mText.length(), "end");
760                     int[] wordIndices = getWordDelta(start, end);
761                     if (classification != null) {
762                         mClassificationSession.onSelectionEvent(
763                                 SelectionEvent.createSelectionActionEvent(
764                                         wordIndices[0], wordIndices[1], action,
765                                         classification));
766                     } else {
767                         mClassificationSession.onSelectionEvent(
768                                 SelectionEvent.createSelectionActionEvent(
769                                         wordIndices[0], wordIndices[1], action));
770                     }
771                     if (SelectionEvent.isTerminal(action)) {
772                         endTextClassificationSession();
773                     }
774                 }
775             } catch (Exception e) {
776                 // Avoid crashes due to logging.
777                 Log.e(LOG_TAG, "" + e.getMessage(), e);
778             }
779         }
780 
isEditTextLogger()781         public boolean isEditTextLogger() {
782             return mEditTextLogger;
783         }
784 
endTextClassificationSession()785         public void endTextClassificationSession() {
786             if (hasActiveClassificationSession()) {
787                 mClassificationSession.destroy();
788             }
789         }
790 
hasActiveClassificationSession()791         private boolean hasActiveClassificationSession() {
792             return mClassificationSession != null && !mClassificationSession.isDestroyed();
793         }
794 
getWordDelta(int start, int end)795         private int[] getWordDelta(int start, int end) {
796             int[] wordIndices = new int[2];
797 
798             if (start == mStartIndex) {
799                 wordIndices[0] = 0;
800             } else if (start < mStartIndex) {
801                 wordIndices[0] = -countWordsForward(start);
802             } else {  // start > mStartIndex
803                 wordIndices[0] = countWordsBackward(start);
804 
805                 // For the selection start index, avoid counting a partial word backwards.
806                 if (!mTokenIterator.isBoundary(start)
807                         && !isWhitespace(
808                         mTokenIterator.preceding(start),
809                         mTokenIterator.following(start))) {
810                     // We counted a partial word. Remove it.
811                     wordIndices[0]--;
812                 }
813             }
814 
815             if (end == mStartIndex) {
816                 wordIndices[1] = 0;
817             } else if (end < mStartIndex) {
818                 wordIndices[1] = -countWordsForward(end);
819             } else {  // end > mStartIndex
820                 wordIndices[1] = countWordsBackward(end);
821             }
822 
823             return wordIndices;
824         }
825 
countWordsBackward(int from)826         private int countWordsBackward(int from) {
827             Preconditions.checkArgument(from >= mStartIndex);
828             int wordCount = 0;
829             int offset = from;
830             while (offset > mStartIndex) {
831                 int start = mTokenIterator.preceding(offset);
832                 if (!isWhitespace(start, offset)) {
833                     wordCount++;
834                 }
835                 offset = start;
836             }
837             return wordCount;
838         }
839 
countWordsForward(int from)840         private int countWordsForward(int from) {
841             Preconditions.checkArgument(from <= mStartIndex);
842             int wordCount = 0;
843             int offset = from;
844             while (offset < mStartIndex) {
845                 int end = mTokenIterator.following(offset);
846                 if (!isWhitespace(offset, end)) {
847                     wordCount++;
848                 }
849                 offset = end;
850             }
851             return wordCount;
852         }
853 
isWhitespace(int start, int end)854         private boolean isWhitespace(int start, int end) {
855             return PATTERN_WHITESPACE.matcher(mText.substring(start, end)).matches();
856         }
857     }
858 
859     /**
860      * AsyncTask for running a query on a background thread and returning the result on the
861      * UiThread. The AsyncTask times out after a specified time, returning a null result if the
862      * query has not yet returned.
863      */
864     private static final class TextClassificationAsyncTask
865             extends AsyncTask<Void, Void, SelectionResult> {
866 
867         private final int mTimeOutDuration;
868         private final Supplier<SelectionResult> mSelectionResultSupplier;
869         private final Consumer<SelectionResult> mSelectionResultCallback;
870         private final Supplier<SelectionResult> mTimeOutResultSupplier;
871         private final TextView mTextView;
872         private final String mOriginalText;
873 
874         /**
875          * @param textView the TextView
876          * @param timeOut time in milliseconds to timeout the query if it has not completed
877          * @param selectionResultSupplier fetches the selection results. Runs on a background thread
878          * @param selectionResultCallback receives the selection results. Runs on the UiThread
879          * @param timeOutResultSupplier default result if the task times out
880          */
TextClassificationAsyncTask( @onNull TextView textView, int timeOut, @NonNull Supplier<SelectionResult> selectionResultSupplier, @NonNull Consumer<SelectionResult> selectionResultCallback, @NonNull Supplier<SelectionResult> timeOutResultSupplier)881         TextClassificationAsyncTask(
882                 @NonNull TextView textView, int timeOut,
883                 @NonNull Supplier<SelectionResult> selectionResultSupplier,
884                 @NonNull Consumer<SelectionResult> selectionResultCallback,
885                 @NonNull Supplier<SelectionResult> timeOutResultSupplier) {
886             super(textView != null ? textView.getHandler() : null);
887             mTextView = Preconditions.checkNotNull(textView);
888             mTimeOutDuration = timeOut;
889             mSelectionResultSupplier = Preconditions.checkNotNull(selectionResultSupplier);
890             mSelectionResultCallback = Preconditions.checkNotNull(selectionResultCallback);
891             mTimeOutResultSupplier = Preconditions.checkNotNull(timeOutResultSupplier);
892             // Make a copy of the original text.
893             mOriginalText = getText(mTextView).toString();
894         }
895 
896         @Override
897         @WorkerThread
doInBackground(Void... params)898         protected SelectionResult doInBackground(Void... params) {
899             final Runnable onTimeOut = this::onTimeOut;
900             mTextView.postDelayed(onTimeOut, mTimeOutDuration);
901             final SelectionResult result = mSelectionResultSupplier.get();
902             mTextView.removeCallbacks(onTimeOut);
903             return result;
904         }
905 
906         @Override
907         @UiThread
onPostExecute(SelectionResult result)908         protected void onPostExecute(SelectionResult result) {
909             result = TextUtils.equals(mOriginalText, getText(mTextView)) ? result : null;
910             mSelectionResultCallback.accept(result);
911         }
912 
onTimeOut()913         private void onTimeOut() {
914             if (getStatus() == Status.RUNNING) {
915                 onPostExecute(mTimeOutResultSupplier.get());
916             }
917             cancel(true);
918         }
919     }
920 
921     /**
922      * Helper class for querying the TextClassifier.
923      * It trims text so that only text necessary to provide context of the selected text is
924      * sent to the TextClassifier.
925      */
926     private static final class TextClassificationHelper {
927 
928         private static final int TRIM_DELTA = 120;  // characters
929 
930         private final Context mContext;
931         private Supplier<TextClassifier> mTextClassifier;
932 
933         /** The original TextView text. **/
934         private String mText;
935         /** Start index relative to mText. */
936         private int mSelectionStart;
937         /** End index relative to mText. */
938         private int mSelectionEnd;
939 
940         @Nullable
941         private LocaleList mDefaultLocales;
942 
943         /** Trimmed text starting from mTrimStart in mText. */
944         private CharSequence mTrimmedText;
945         /** Index indicating the start of mTrimmedText in mText. */
946         private int mTrimStart;
947         /** Start index relative to mTrimmedText */
948         private int mRelativeStart;
949         /** End index relative to mTrimmedText */
950         private int mRelativeEnd;
951 
952         /** Information about the last classified text to avoid re-running a query. */
953         private CharSequence mLastClassificationText;
954         private int mLastClassificationSelectionStart;
955         private int mLastClassificationSelectionEnd;
956         private LocaleList mLastClassificationLocales;
957         private SelectionResult mLastClassificationResult;
958 
959         /** Whether the TextClassifier has been initialized. */
960         private boolean mHot;
961 
TextClassificationHelper(Context context, Supplier<TextClassifier> textClassifier, CharSequence text, int selectionStart, int selectionEnd, LocaleList locales)962         TextClassificationHelper(Context context, Supplier<TextClassifier> textClassifier,
963                 CharSequence text, int selectionStart, int selectionEnd, LocaleList locales) {
964             init(textClassifier, text, selectionStart, selectionEnd, locales);
965             mContext = Preconditions.checkNotNull(context);
966         }
967 
968         @UiThread
init(Supplier<TextClassifier> textClassifier, CharSequence text, int selectionStart, int selectionEnd, LocaleList locales)969         public void init(Supplier<TextClassifier> textClassifier, CharSequence text,
970                 int selectionStart, int selectionEnd, LocaleList locales) {
971             mTextClassifier = Preconditions.checkNotNull(textClassifier);
972             mText = Preconditions.checkNotNull(text).toString();
973             mLastClassificationText = null; // invalidate.
974             Preconditions.checkArgument(selectionEnd > selectionStart);
975             mSelectionStart = selectionStart;
976             mSelectionEnd = selectionEnd;
977             mDefaultLocales = locales;
978         }
979 
980         @WorkerThread
classifyText()981         public SelectionResult classifyText() {
982             mHot = true;
983             return performClassification(null /* selection */);
984         }
985 
986         @WorkerThread
suggestSelection()987         public SelectionResult suggestSelection() {
988             mHot = true;
989             trimText();
990             final TextSelection selection;
991             if (mContext.getApplicationInfo().targetSdkVersion >= Build.VERSION_CODES.P) {
992                 final TextSelection.Request request = new TextSelection.Request.Builder(
993                         mTrimmedText, mRelativeStart, mRelativeEnd)
994                         .setDefaultLocales(mDefaultLocales)
995                         .setDarkLaunchAllowed(true)
996                         .build();
997                 selection = mTextClassifier.get().suggestSelection(request);
998             } else {
999                 // Use old APIs.
1000                 selection = mTextClassifier.get().suggestSelection(
1001                         mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales);
1002             }
1003             // Do not classify new selection boundaries if TextClassifier should be dark launched.
1004             if (!isDarkLaunchEnabled()) {
1005                 mSelectionStart = Math.max(0, selection.getSelectionStartIndex() + mTrimStart);
1006                 mSelectionEnd = Math.min(
1007                         mText.length(), selection.getSelectionEndIndex() + mTrimStart);
1008             }
1009             return performClassification(selection);
1010         }
1011 
getOriginalSelection()1012         public SelectionResult getOriginalSelection() {
1013             return new SelectionResult(mSelectionStart, mSelectionEnd, null, null);
1014         }
1015 
1016         /**
1017          * Maximum time (in milliseconds) to wait for a textclassifier result before timing out.
1018          */
1019         // TODO: Consider making this a ViewConfiguration.
getTimeoutDuration()1020         public int getTimeoutDuration() {
1021             if (mHot) {
1022                 return 200;
1023             } else {
1024                 // Return a slightly larger number than usual when the TextClassifier is first
1025                 // initialized. Initialization would usually take longer than subsequent calls to
1026                 // the TextClassifier. The impact of this on the UI is that we do not show the
1027                 // selection handles or toolbar until after this timeout.
1028                 return 500;
1029             }
1030         }
1031 
isDarkLaunchEnabled()1032         private boolean isDarkLaunchEnabled() {
1033             return TextClassificationManager.getSettings(mContext).isModelDarkLaunchEnabled();
1034         }
1035 
performClassification(@ullable TextSelection selection)1036         private SelectionResult performClassification(@Nullable TextSelection selection) {
1037             if (!Objects.equals(mText, mLastClassificationText)
1038                     || mSelectionStart != mLastClassificationSelectionStart
1039                     || mSelectionEnd != mLastClassificationSelectionEnd
1040                     || !Objects.equals(mDefaultLocales, mLastClassificationLocales)) {
1041 
1042                 mLastClassificationText = mText;
1043                 mLastClassificationSelectionStart = mSelectionStart;
1044                 mLastClassificationSelectionEnd = mSelectionEnd;
1045                 mLastClassificationLocales = mDefaultLocales;
1046 
1047                 trimText();
1048                 final TextClassification classification;
1049                 if (Linkify.containsUnsupportedCharacters(mText)) {
1050                     // Do not show smart actions for text containing unsupported characters.
1051                     android.util.EventLog.writeEvent(0x534e4554, "116321860", -1, "");
1052                     classification = TextClassification.EMPTY;
1053                 } else if (mContext.getApplicationInfo().targetSdkVersion
1054                         >= Build.VERSION_CODES.P) {
1055                     final TextClassification.Request request =
1056                             new TextClassification.Request.Builder(
1057                                     mTrimmedText, mRelativeStart, mRelativeEnd)
1058                                     .setDefaultLocales(mDefaultLocales)
1059                                     .build();
1060                     classification = mTextClassifier.get().classifyText(request);
1061                 } else {
1062                     // Use old APIs.
1063                     classification = mTextClassifier.get().classifyText(
1064                             mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales);
1065                 }
1066                 mLastClassificationResult = new SelectionResult(
1067                         mSelectionStart, mSelectionEnd, classification, selection);
1068 
1069             }
1070             return mLastClassificationResult;
1071         }
1072 
trimText()1073         private void trimText() {
1074             mTrimStart = Math.max(0, mSelectionStart - TRIM_DELTA);
1075             final int referenceEnd = Math.min(mText.length(), mSelectionEnd + TRIM_DELTA);
1076             mTrimmedText = mText.subSequence(mTrimStart, referenceEnd);
1077             mRelativeStart = mSelectionStart - mTrimStart;
1078             mRelativeEnd = mSelectionEnd - mTrimStart;
1079         }
1080     }
1081 
1082     /**
1083      * Selection result.
1084      */
1085     private static final class SelectionResult {
1086         private final int mStart;
1087         private final int mEnd;
1088         @Nullable private final TextClassification mClassification;
1089         @Nullable private final TextSelection mSelection;
1090 
SelectionResult(int start, int end, @Nullable TextClassification classification, @Nullable TextSelection selection)1091         SelectionResult(int start, int end,
1092                 @Nullable TextClassification classification, @Nullable TextSelection selection) {
1093             mStart = start;
1094             mEnd = end;
1095             mClassification = classification;
1096             mSelection = selection;
1097         }
1098     }
1099 
1100     @SelectionEvent.ActionType
getActionType(int menuItemId)1101     private static int getActionType(int menuItemId) {
1102         switch (menuItemId) {
1103             case TextView.ID_SELECT_ALL:
1104                 return SelectionEvent.ACTION_SELECT_ALL;
1105             case TextView.ID_CUT:
1106                 return SelectionEvent.ACTION_CUT;
1107             case TextView.ID_COPY:
1108                 return SelectionEvent.ACTION_COPY;
1109             case TextView.ID_PASTE:  // fall through
1110             case TextView.ID_PASTE_AS_PLAIN_TEXT:
1111                 return SelectionEvent.ACTION_PASTE;
1112             case TextView.ID_SHARE:
1113                 return SelectionEvent.ACTION_SHARE;
1114             case TextView.ID_ASSIST:
1115                 return SelectionEvent.ACTION_SMART_SHARE;
1116             default:
1117                 return SelectionEvent.ACTION_OTHER;
1118         }
1119     }
1120 
getText(TextView textView)1121     private static CharSequence getText(TextView textView) {
1122         // Extracts the textView's text.
1123         // TODO: Investigate why/when TextView.getText() is null.
1124         final CharSequence text = textView.getText();
1125         if (text != null) {
1126             return text;
1127         }
1128         return "";
1129     }
1130 }
1131