1 /** 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.ext.services.notification; 17 18 import android.annotation.Nullable; 19 import android.app.Notification; 20 import android.app.PendingIntent; 21 import android.app.Person; 22 import android.app.RemoteAction; 23 import android.app.RemoteInput; 24 import android.content.Context; 25 import android.content.Intent; 26 import android.ext.services.R; 27 import android.graphics.drawable.Icon; 28 import android.os.Bundle; 29 import android.os.Parcelable; 30 import android.os.Process; 31 import android.service.notification.NotificationAssistantService; 32 import android.text.TextUtils; 33 import android.util.ArrayMap; 34 import android.util.LruCache; 35 import android.util.Pair; 36 import android.view.textclassifier.ConversationAction; 37 import android.view.textclassifier.ConversationActions; 38 import android.view.textclassifier.TextClassificationContext; 39 import android.view.textclassifier.TextClassificationManager; 40 import android.view.textclassifier.TextClassifier; 41 import android.view.textclassifier.TextClassifierEvent; 42 43 import com.android.internal.util.ArrayUtils; 44 45 import java.time.Instant; 46 import java.time.ZoneOffset; 47 import java.time.ZonedDateTime; 48 import java.util.ArrayDeque; 49 import java.util.ArrayList; 50 import java.util.Collections; 51 import java.util.Deque; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.Objects; 55 56 /** 57 * Generates suggestions from incoming notifications. 58 * 59 * Methods in this class should be called in a single worker thread. 60 */ 61 public class SmartActionsHelper { 62 static final String ENTITIES_EXTRAS = "entities-extras"; 63 static final String KEY_ACTION_TYPE = "action_type"; 64 static final String KEY_ACTION_SCORE = "action_score"; 65 static final String KEY_TEXT = "text"; 66 // If a notification has any of these flags set, it's inelgibile for actions being added. 67 private static final int FLAG_MASK_INELGIBILE_FOR_ACTIONS = 68 Notification.FLAG_ONGOING_EVENT 69 | Notification.FLAG_FOREGROUND_SERVICE 70 | Notification.FLAG_GROUP_SUMMARY 71 | Notification.FLAG_NO_CLEAR; 72 private static final int MAX_RESULT_ID_TO_CACHE = 20; 73 74 private static final List<String> HINTS = 75 Collections.singletonList(ConversationActions.Request.HINT_FOR_NOTIFICATION); 76 private static final ConversationActions EMPTY_CONVERSATION_ACTIONS = 77 new ConversationActions(Collections.emptyList(), null); 78 79 private Context mContext; 80 private TextClassificationManager mTextClassificationManager; 81 private AssistantSettings mSettings; 82 private LruCache<String, Session> mSessionCache = new LruCache<>(MAX_RESULT_ID_TO_CACHE); 83 SmartActionsHelper(Context context, AssistantSettings settings)84 SmartActionsHelper(Context context, AssistantSettings settings) { 85 mContext = context; 86 mTextClassificationManager = mContext.getSystemService(TextClassificationManager.class); 87 mSettings = settings; 88 } 89 suggest(NotificationEntry entry)90 SmartSuggestions suggest(NotificationEntry entry) { 91 // Whenever suggest() is called on a notification, its previous session is ended. 92 mSessionCache.remove(entry.getSbn().getKey()); 93 94 boolean eligibleForReplyAdjustment = 95 mSettings.mGenerateReplies && isEligibleForReplyAdjustment(entry); 96 boolean eligibleForActionAdjustment = 97 mSettings.mGenerateActions && isEligibleForActionAdjustment(entry); 98 99 ConversationActions conversationActionsResult = 100 suggestConversationActions( 101 entry, 102 eligibleForReplyAdjustment, 103 eligibleForActionAdjustment); 104 105 String resultId = conversationActionsResult.getId(); 106 List<ConversationAction> conversationActions = 107 conversationActionsResult.getConversationActions(); 108 109 ArrayList<CharSequence> replies = new ArrayList<>(); 110 Map<CharSequence, Float> repliesScore = new ArrayMap<>(); 111 for (ConversationAction conversationAction : conversationActions) { 112 CharSequence textReply = conversationAction.getTextReply(); 113 if (TextUtils.isEmpty(textReply)) { 114 continue; 115 } 116 replies.add(textReply); 117 repliesScore.put(textReply, conversationAction.getConfidenceScore()); 118 } 119 120 ArrayList<Notification.Action> actions = new ArrayList<>(); 121 for (ConversationAction conversationAction : conversationActions) { 122 if (!TextUtils.isEmpty(conversationAction.getTextReply())) { 123 continue; 124 } 125 Notification.Action notificationAction; 126 if (conversationAction.getAction() == null) { 127 notificationAction = 128 createNotificationActionWithoutRemoteAction(conversationAction); 129 } else { 130 notificationAction = createNotificationActionFromRemoteAction( 131 conversationAction.getAction(), 132 conversationAction.getType(), 133 conversationAction.getConfidenceScore()); 134 } 135 if (notificationAction != null) { 136 actions.add(notificationAction); 137 } 138 } 139 140 // Start a new session for logging if necessary. 141 if (!TextUtils.isEmpty(resultId) 142 && !conversationActions.isEmpty() 143 && suggestionsMightBeUsedInNotification( 144 entry, !actions.isEmpty(), !replies.isEmpty())) { 145 mSessionCache.put(entry.getSbn().getKey(), new Session(resultId, repliesScore)); 146 } 147 148 return new SmartSuggestions(replies, actions); 149 } 150 151 /** 152 * Creates notification action from ConversationAction that does not come up a RemoteAction. 153 * It could happen because we don't have common intents for some actions, like copying text. 154 */ 155 @Nullable createNotificationActionWithoutRemoteAction( ConversationAction conversationAction)156 private Notification.Action createNotificationActionWithoutRemoteAction( 157 ConversationAction conversationAction) { 158 if (ConversationAction.TYPE_COPY.equals(conversationAction.getType())) { 159 return createCopyCodeAction(conversationAction); 160 } 161 return null; 162 } 163 164 @Nullable createCopyCodeAction(ConversationAction conversationAction)165 private Notification.Action createCopyCodeAction(ConversationAction conversationAction) { 166 Bundle extras = conversationAction.getExtras(); 167 if (extras == null) { 168 return null; 169 } 170 Bundle entitiesExtas = extras.getParcelable(ENTITIES_EXTRAS); 171 if (entitiesExtas == null) { 172 return null; 173 } 174 String code = entitiesExtas.getString(KEY_TEXT); 175 if (TextUtils.isEmpty(code)) { 176 return null; 177 } 178 String contentDescription = mContext.getString(R.string.copy_code_desc, code); 179 Intent intent = new Intent(mContext, CopyCodeActivity.class); 180 intent.putExtra(Intent.EXTRA_TEXT, code); 181 182 RemoteAction remoteAction = new RemoteAction(Icon.createWithResource( 183 mContext.getResources(), 184 com.android.internal.R.drawable.ic_menu_copy_material), 185 code, 186 contentDescription, 187 PendingIntent.getActivity( 188 mContext, 189 code.hashCode(), 190 intent, 191 PendingIntent.FLAG_UPDATE_CURRENT 192 )); 193 194 return createNotificationActionFromRemoteAction( 195 remoteAction, 196 ConversationAction.TYPE_COPY, 197 conversationAction.getConfidenceScore()); 198 } 199 200 /** 201 * Returns whether the suggestion might be used in the notifications in SysUI. 202 * <p> 203 * Currently, NAS has no idea if suggestions will actually be used in the notification, and thus 204 * this function tries to make a heuristic. This function tries to optimize the precision, 205 * that means when it is unsure, it will return false. The objective is to avoid false positive, 206 * which could pollute the log and CTR as we are logging click rate of suggestions that could 207 * be never visible to users. On the other hand, it is fine to have false negative because 208 * it would be just like sampling. 209 */ suggestionsMightBeUsedInNotification( NotificationEntry notificationEntry, boolean hasSmartAction, boolean hasSmartReply)210 private boolean suggestionsMightBeUsedInNotification( 211 NotificationEntry notificationEntry, boolean hasSmartAction, boolean hasSmartReply) { 212 Notification notification = notificationEntry.getNotification(); 213 boolean hasAppGeneratedContextualActions = !notification.getContextualActions().isEmpty(); 214 215 Pair<RemoteInput, Notification.Action> freeformRemoteInputAndAction = 216 notification.findRemoteInputActionPair(/* requiresFreeform */ true); 217 boolean hasAppGeneratedReplies = false; 218 boolean allowGeneratedReplies = false; 219 if (freeformRemoteInputAndAction != null) { 220 RemoteInput freeformRemoteInput = freeformRemoteInputAndAction.first; 221 Notification.Action actionWithFreeformRemoteInput = freeformRemoteInputAndAction.second; 222 hasAppGeneratedReplies = !ArrayUtils.isEmpty(freeformRemoteInput.getChoices()); 223 allowGeneratedReplies = actionWithFreeformRemoteInput.getAllowGeneratedReplies(); 224 } 225 226 if (hasAppGeneratedReplies || hasAppGeneratedContextualActions) { 227 return false; 228 } 229 return hasSmartAction && notification.getAllowSystemGeneratedContextualActions() 230 || hasSmartReply && allowGeneratedReplies; 231 } 232 reportActionsGenerated( String resultId, List<ConversationAction> conversationActions)233 private void reportActionsGenerated( 234 String resultId, List<ConversationAction> conversationActions) { 235 if (TextUtils.isEmpty(resultId)) { 236 return; 237 } 238 TextClassifierEvent textClassifierEvent = 239 createTextClassifierEventBuilder( 240 TextClassifierEvent.TYPE_ACTIONS_GENERATED, resultId) 241 .setEntityTypes(conversationActions.stream() 242 .map(ConversationAction::getType) 243 .toArray(String[]::new)) 244 .build(); 245 getTextClassifier().onTextClassifierEvent(textClassifierEvent); 246 } 247 248 /** 249 * Adds action adjustments based on the notification contents. 250 */ suggestConversationActions( NotificationEntry entry, boolean includeReplies, boolean includeActions)251 private ConversationActions suggestConversationActions( 252 NotificationEntry entry, 253 boolean includeReplies, 254 boolean includeActions) { 255 if (!includeReplies && !includeActions) { 256 return EMPTY_CONVERSATION_ACTIONS; 257 } 258 List<ConversationActions.Message> messages = extractMessages(entry.getNotification()); 259 if (messages.isEmpty()) { 260 return EMPTY_CONVERSATION_ACTIONS; 261 } 262 // Do not generate smart actions if the last message is from the local user. 263 ConversationActions.Message lastMessage = messages.get(messages.size() - 1); 264 if (arePersonsEqual( 265 ConversationActions.Message.PERSON_USER_SELF, lastMessage.getAuthor())) { 266 return EMPTY_CONVERSATION_ACTIONS; 267 } 268 269 TextClassifier.EntityConfig.Builder typeConfigBuilder = 270 new TextClassifier.EntityConfig.Builder(); 271 if (!includeReplies) { 272 typeConfigBuilder.setExcludedTypes( 273 Collections.singletonList(ConversationAction.TYPE_TEXT_REPLY)); 274 } else if (!includeActions) { 275 typeConfigBuilder 276 .setIncludedTypes( 277 Collections.singletonList(ConversationAction.TYPE_TEXT_REPLY)) 278 .includeTypesFromTextClassifier(false); 279 } 280 ConversationActions.Request request = 281 new ConversationActions.Request.Builder(messages) 282 .setMaxSuggestions(mSettings.mMaxSuggestions) 283 .setHints(HINTS) 284 .setTypeConfig(typeConfigBuilder.build()) 285 .build(); 286 ConversationActions conversationActions = 287 getTextClassifier().suggestConversationActions(request); 288 reportActionsGenerated( 289 conversationActions.getId(), conversationActions.getConversationActions()); 290 return conversationActions; 291 } 292 onNotificationExpansionChanged(NotificationEntry entry, boolean isExpanded)293 void onNotificationExpansionChanged(NotificationEntry entry, boolean isExpanded) { 294 if (!isExpanded) { 295 return; 296 } 297 Session session = mSessionCache.get(entry.getSbn().getKey()); 298 if (session == null) { 299 return; 300 } 301 // Only report if this is the first time the user sees these suggestions. 302 if (entry.isShowActionEventLogged()) { 303 return; 304 } 305 entry.setShowActionEventLogged(); 306 TextClassifierEvent textClassifierEvent = 307 createTextClassifierEventBuilder( 308 TextClassifierEvent.TYPE_ACTIONS_SHOWN, session.resultId) 309 .build(); 310 // TODO: If possible, report which replies / actions are actually seen by user. 311 getTextClassifier().onTextClassifierEvent(textClassifierEvent); 312 } 313 onNotificationDirectReplied(String key)314 void onNotificationDirectReplied(String key) { 315 Session session = mSessionCache.get(key); 316 if (session == null) { 317 return; 318 } 319 TextClassifierEvent textClassifierEvent = 320 createTextClassifierEventBuilder( 321 TextClassifierEvent.TYPE_MANUAL_REPLY, session.resultId) 322 .build(); 323 getTextClassifier().onTextClassifierEvent(textClassifierEvent); 324 } 325 onSuggestedReplySent(String key, CharSequence reply, @NotificationAssistantService.Source int source)326 void onSuggestedReplySent(String key, CharSequence reply, 327 @NotificationAssistantService.Source int source) { 328 if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) { 329 return; 330 } 331 Session session = mSessionCache.get(key); 332 if (session == null) { 333 return; 334 } 335 TextClassifierEvent textClassifierEvent = 336 createTextClassifierEventBuilder( 337 TextClassifierEvent.TYPE_SMART_ACTION, session.resultId) 338 .setEntityTypes(ConversationAction.TYPE_TEXT_REPLY) 339 .setScores(session.repliesScores.getOrDefault(reply, 0f)) 340 .build(); 341 getTextClassifier().onTextClassifierEvent(textClassifierEvent); 342 } 343 onActionClicked(String key, Notification.Action action, @NotificationAssistantService.Source int source)344 void onActionClicked(String key, Notification.Action action, 345 @NotificationAssistantService.Source int source) { 346 if (source != NotificationAssistantService.SOURCE_FROM_ASSISTANT) { 347 return; 348 } 349 Session session = mSessionCache.get(key); 350 if (session == null) { 351 return; 352 } 353 String actionType = action.getExtras().getString(KEY_ACTION_TYPE); 354 if (actionType == null) { 355 return; 356 } 357 TextClassifierEvent textClassifierEvent = 358 createTextClassifierEventBuilder( 359 TextClassifierEvent.TYPE_SMART_ACTION, session.resultId) 360 .setEntityTypes(actionType) 361 .build(); 362 getTextClassifier().onTextClassifierEvent(textClassifierEvent); 363 } 364 createNotificationActionFromRemoteAction( RemoteAction remoteAction, String actionType, float score)365 private Notification.Action createNotificationActionFromRemoteAction( 366 RemoteAction remoteAction, String actionType, float score) { 367 Icon icon = remoteAction.shouldShowIcon() 368 ? remoteAction.getIcon() 369 : Icon.createWithResource(mContext, com.android.internal.R.drawable.ic_action_open); 370 Bundle extras = new Bundle(); 371 extras.putString(KEY_ACTION_TYPE, actionType); 372 extras.putFloat(KEY_ACTION_SCORE, score); 373 return new Notification.Action.Builder( 374 icon, 375 remoteAction.getTitle(), 376 remoteAction.getActionIntent()) 377 .setContextual(true) 378 .addExtras(extras) 379 .build(); 380 } 381 createTextClassifierEventBuilder( int eventType, String resultId)382 private TextClassifierEvent.ConversationActionsEvent.Builder createTextClassifierEventBuilder( 383 int eventType, String resultId) { 384 return new TextClassifierEvent.ConversationActionsEvent.Builder(eventType) 385 .setEventContext( 386 new TextClassificationContext.Builder( 387 mContext.getPackageName(), TextClassifier.WIDGET_TYPE_NOTIFICATION) 388 .build()) 389 .setResultId(resultId); 390 } 391 392 /** 393 * Returns whether a notification is eligible for action adjustments. 394 * 395 * <p>We exclude system notifications, those that get refreshed frequently, or ones that relate 396 * to fundamental phone functionality where any error would result in a very negative user 397 * experience. 398 */ isEligibleForActionAdjustment(NotificationEntry entry)399 private boolean isEligibleForActionAdjustment(NotificationEntry entry) { 400 Notification notification = entry.getNotification(); 401 String pkg = entry.getSbn().getPackageName(); 402 if (!Process.myUserHandle().equals(entry.getSbn().getUser())) { 403 return false; 404 } 405 if ((notification.flags & FLAG_MASK_INELGIBILE_FOR_ACTIONS) != 0) { 406 return false; 407 } 408 if (TextUtils.isEmpty(pkg) || pkg.equals("android")) { 409 return false; 410 } 411 // For now, we are only interested in messages. 412 return entry.isMessaging(); 413 } 414 isEligibleForReplyAdjustment(NotificationEntry entry)415 private boolean isEligibleForReplyAdjustment(NotificationEntry entry) { 416 if (!Process.myUserHandle().equals(entry.getSbn().getUser())) { 417 return false; 418 } 419 String pkg = entry.getSbn().getPackageName(); 420 if (TextUtils.isEmpty(pkg) || pkg.equals("android")) { 421 return false; 422 } 423 // For now, we are only interested in messages. 424 if (!entry.isMessaging()) { 425 return false; 426 } 427 // Does not make sense to provide suggested replies if it is not something that can be 428 // replied. 429 if (!entry.hasInlineReply()) { 430 return false; 431 } 432 return true; 433 } 434 435 /** Returns the text most salient for action extraction in a notification. */ extractMessages(Notification notification)436 private List<ConversationActions.Message> extractMessages(Notification notification) { 437 Parcelable[] messages = notification.extras.getParcelableArray(Notification.EXTRA_MESSAGES); 438 if (messages == null || messages.length == 0) { 439 return Collections.singletonList(new ConversationActions.Message.Builder( 440 ConversationActions.Message.PERSON_USER_OTHERS) 441 .setText(notification.extras.getCharSequence(Notification.EXTRA_TEXT)) 442 .build()); 443 } 444 Person localUser = notification.extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON); 445 Deque<ConversationActions.Message> extractMessages = new ArrayDeque<>(); 446 for (int i = messages.length - 1; i >= 0; i--) { 447 Notification.MessagingStyle.Message message = 448 Notification.MessagingStyle.Message.getMessageFromBundle((Bundle) messages[i]); 449 if (message == null) { 450 continue; 451 } 452 // As per the javadoc of Notification.addMessage, null means local user. 453 Person senderPerson = message.getSenderPerson(); 454 if (senderPerson == null) { 455 senderPerson = localUser; 456 } 457 Person author = localUser != null && arePersonsEqual(localUser, senderPerson) 458 ? ConversationActions.Message.PERSON_USER_SELF : senderPerson; 459 extractMessages.push(new ConversationActions.Message.Builder(author) 460 .setText(message.getText()) 461 .setReferenceTime( 462 ZonedDateTime.ofInstant(Instant.ofEpochMilli(message.getTimestamp()), 463 ZoneOffset.systemDefault())) 464 .build()); 465 if (extractMessages.size() >= mSettings.mMaxMessagesToExtract) { 466 break; 467 } 468 } 469 return new ArrayList<>(extractMessages); 470 } 471 getTextClassifier()472 private TextClassifier getTextClassifier() { 473 return mTextClassificationManager.getTextClassifier(); 474 } 475 arePersonsEqual(Person left, Person right)476 private static boolean arePersonsEqual(Person left, Person right) { 477 return Objects.equals(left.getKey(), right.getKey()) 478 && Objects.equals(left.getName(), right.getName()) 479 && Objects.equals(left.getUri(), right.getUri()); 480 } 481 482 static class SmartSuggestions { 483 public final ArrayList<CharSequence> replies; 484 public final ArrayList<Notification.Action> actions; 485 SmartSuggestions( ArrayList<CharSequence> replies, ArrayList<Notification.Action> actions)486 SmartSuggestions( 487 ArrayList<CharSequence> replies, ArrayList<Notification.Action> actions) { 488 this.replies = replies; 489 this.actions = actions; 490 } 491 } 492 493 private static class Session { 494 public final String resultId; 495 public final Map<CharSequence, Float> repliesScores; 496 Session(String resultId, Map<CharSequence, Float> repliesScores)497 Session(String resultId, Map<CharSequence, Float> repliesScores) { 498 this.resultId = resultId; 499 this.repliesScores = repliesScores; 500 } 501 } 502 } 503