1 package com.android.systemui.assist; 2 3 import static com.android.systemui.DejankUtils.whitelistIpcs; 4 import static com.android.systemui.shared.system.QuickStepContract.SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED; 5 6 import android.annotation.NonNull; 7 import android.annotation.Nullable; 8 import android.app.ActivityManager; 9 import android.app.ActivityOptions; 10 import android.app.SearchManager; 11 import android.app.StatusBarManager; 12 import android.content.ActivityNotFoundException; 13 import android.content.ComponentName; 14 import android.content.Context; 15 import android.content.Intent; 16 import android.metrics.LogMaker; 17 import android.os.AsyncTask; 18 import android.os.Bundle; 19 import android.os.Handler; 20 import android.os.RemoteException; 21 import android.os.SystemClock; 22 import android.os.UserHandle; 23 import android.provider.Settings; 24 import android.service.voice.VisualQueryAttentionResult; 25 import android.service.voice.VoiceInteractionSession; 26 import android.util.Log; 27 import android.view.WindowManager; 28 29 import com.android.internal.app.AssistUtils; 30 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 31 import com.android.internal.app.IVisualQueryRecognitionStatusListener; 32 import com.android.internal.app.IVoiceInteractionSessionListener; 33 import com.android.internal.logging.MetricsLogger; 34 import com.android.internal.logging.nano.MetricsProto.MetricsEvent; 35 import com.android.systemui.assist.domain.interactor.AssistInteractor; 36 import com.android.systemui.assist.ui.DefaultUiController; 37 import com.android.systemui.dagger.SysUISingleton; 38 import com.android.systemui.dagger.qualifiers.Main; 39 import com.android.systemui.model.SysUiState; 40 import com.android.systemui.recents.LauncherProxyService; 41 import com.android.systemui.res.R; 42 import com.android.systemui.settings.DisplayTracker; 43 import com.android.systemui.settings.UserTracker; 44 import com.android.systemui.statusbar.CommandQueue; 45 import com.android.systemui.statusbar.policy.DeviceProvisionedController; 46 import com.android.systemui.user.domain.interactor.SelectedUserInteractor; 47 import com.android.systemui.util.settings.SecureSettings; 48 49 import dagger.Lazy; 50 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.List; 54 55 import javax.inject.Inject; 56 57 /** 58 * Class to manage everything related to assist in SystemUI. 59 */ 60 @SysUISingleton 61 public class AssistManager { 62 63 /** 64 * Controls the UI for showing Assistant invocation progress. 65 */ 66 public interface UiController { 67 /** 68 * Updates the invocation progress. 69 * 70 * @param type one of INVOCATION_TYPE_GESTURE, INVOCATION_TYPE_ACTIVE_EDGE, 71 * INVOCATION_TYPE_VOICE, INVOCATION_TYPE_QUICK_SEARCH_BAR, 72 * INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS 73 * @param progress a float between 0 and 1 inclusive. 0 represents the beginning of the 74 * gesture; 1 represents the end. 75 */ onInvocationProgress(int type, float progress)76 void onInvocationProgress(int type, float progress); 77 78 /** 79 * Called when an invocation gesture completes. 80 * 81 * @param velocity the speed of the invocation gesture, in pixels per millisecond. For 82 * drags, this is 0. 83 */ onGestureCompletion(float velocity)84 void onGestureCompletion(float velocity); 85 86 /** 87 * Hides any SysUI for the assistant, but _does not_ close the assistant itself. 88 */ hide()89 void hide(); 90 } 91 92 /** 93 * An interface for a listener that receives notification that visual query attention has 94 * either been gained or lost. 95 */ 96 public interface VisualQueryAttentionListener { 97 /** Called when visual query attention has been gained. */ onAttentionGained()98 void onAttentionGained(); 99 100 /** Called when visual query attention has been lost. */ onAttentionLost()101 void onAttentionLost(); 102 } 103 104 private static final String TAG = "AssistManager"; 105 106 // Note that VERBOSE logging may leak PII (e.g. transcription contents). 107 private static final boolean VERBOSE = false; 108 109 private static final String INVOCATION_TIME_MS_KEY = "invocation_time_ms"; 110 private static final String INVOCATION_PHONE_STATE_KEY = "invocation_phone_state"; 111 protected static final String ACTION_KEY = "action"; 112 protected static final String SET_ASSIST_GESTURE_CONSTRAINED_ACTION = 113 "set_assist_gesture_constrained"; 114 protected static final String CONSTRAINED_KEY = "should_constrain"; 115 116 public static final String INVOCATION_TYPE_KEY = "invocation_type"; 117 public static final int INVOCATION_TYPE_UNKNOWN = 118 AssistUtils.INVOCATION_TYPE_UNKNOWN; 119 public static final int INVOCATION_TYPE_GESTURE = 120 AssistUtils.INVOCATION_TYPE_GESTURE; 121 public static final int INVOCATION_TYPE_OTHER = 122 AssistUtils.INVOCATION_TYPE_PHYSICAL_GESTURE; 123 public static final int INVOCATION_TYPE_VOICE = 124 AssistUtils.INVOCATION_TYPE_VOICE; 125 public static final int INVOCATION_TYPE_QUICK_SEARCH_BAR = 126 AssistUtils.INVOCATION_TYPE_QUICK_SEARCH_BAR; 127 public static final int INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS = 128 AssistUtils.INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS; 129 public static final int INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS = 130 AssistUtils.INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS; 131 public static final int INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS = 132 AssistUtils.INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS; 133 public static final int INVOCATION_TYPE_LAUNCHER_SYSTEM_SHORTCUT = 134 AssistUtils.INVOCATION_TYPE_LAUNCHER_SYSTEM_SHORTCUT; 135 136 public static final int DISMISS_REASON_INVOCATION_CANCELLED = 1; 137 public static final int DISMISS_REASON_TAP = 2; 138 public static final int DISMISS_REASON_BACK = 3; 139 public static final int DISMISS_REASON_TIMEOUT = 4; 140 141 private static final long TIMEOUT_SERVICE = 2500; 142 private static final long TIMEOUT_ACTIVITY = 1000; 143 144 protected final Context mContext; 145 private final AssistDisclosure mAssistDisclosure; 146 private final PhoneStateMonitor mPhoneStateMonitor; 147 private final LauncherProxyService mLauncherProxyService; 148 private final UiController mUiController; 149 protected final Lazy<SysUiState> mSysUiState; 150 protected final AssistLogger mAssistLogger; 151 private final UserTracker mUserTracker; 152 private final DisplayTracker mDisplayTracker; 153 private final SecureSettings mSecureSettings; 154 private final SelectedUserInteractor mSelectedUserInteractor; 155 private final ActivityManager mActivityManager; 156 private final AssistInteractor mInteractor; 157 158 private final DeviceProvisionedController mDeviceProvisionedController; 159 160 private final List<VisualQueryAttentionListener> mVisualQueryAttentionListeners = 161 new ArrayList<>(); 162 163 private final IVisualQueryDetectionAttentionListener mVisualQueryDetectionAttentionListener = 164 new IVisualQueryDetectionAttentionListener.Stub() { 165 @Override 166 public void onAttentionGained(VisualQueryAttentionResult attentionResult) { 167 // TODO (b/319132184): Implemented this with different types. 168 handleVisualAttentionChanged(true); 169 } 170 171 @Override 172 public void onAttentionLost(int interactionIntention) { 173 //TODO (b/319132184): Implemented this with different types. 174 handleVisualAttentionChanged(false); 175 } 176 }; 177 178 private final CommandQueue mCommandQueue; 179 protected final AssistUtils mAssistUtils; 180 181 // Invocation types that should be sent over LauncherProxy instead of handled here. 182 private int[] mAssistOverrideInvocationTypes; 183 184 @Inject AssistManager( DeviceProvisionedController controller, Context context, AssistUtils assistUtils, CommandQueue commandQueue, PhoneStateMonitor phoneStateMonitor, LauncherProxyService launcherProxyService, Lazy<SysUiState> sysUiState, DefaultUiController defaultUiController, AssistLogger assistLogger, @Main Handler uiHandler, UserTracker userTracker, DisplayTracker displayTracker, SecureSettings secureSettings, SelectedUserInteractor selectedUserInteractor, ActivityManager activityManager, AssistInteractor interactor, WindowManager windowManager)185 public AssistManager( 186 DeviceProvisionedController controller, 187 Context context, 188 AssistUtils assistUtils, 189 CommandQueue commandQueue, 190 PhoneStateMonitor phoneStateMonitor, 191 LauncherProxyService launcherProxyService, 192 Lazy<SysUiState> sysUiState, 193 DefaultUiController defaultUiController, 194 AssistLogger assistLogger, 195 @Main Handler uiHandler, 196 UserTracker userTracker, 197 DisplayTracker displayTracker, 198 SecureSettings secureSettings, 199 SelectedUserInteractor selectedUserInteractor, 200 ActivityManager activityManager, 201 AssistInteractor interactor, 202 WindowManager windowManager) { 203 mContext = context; 204 mDeviceProvisionedController = controller; 205 mCommandQueue = commandQueue; 206 mAssistUtils = assistUtils; 207 mAssistDisclosure = new AssistDisclosure(context, uiHandler, windowManager); 208 mLauncherProxyService = launcherProxyService; 209 mPhoneStateMonitor = phoneStateMonitor; 210 mAssistLogger = assistLogger; 211 mUserTracker = userTracker; 212 mDisplayTracker = displayTracker; 213 mSecureSettings = secureSettings; 214 mSelectedUserInteractor = selectedUserInteractor; 215 mActivityManager = activityManager; 216 mInteractor = interactor; 217 218 registerVoiceInteractionSessionListener(); 219 registerVisualQueryRecognitionStatusListener(); 220 221 mUiController = defaultUiController; 222 223 mSysUiState = sysUiState; 224 225 mLauncherProxyService.addCallback(new LauncherProxyService.LauncherProxyListener() { 226 @Override 227 public void onAssistantProgress(float progress) { 228 // Progress goes from 0 to 1 to indicate how close the assist gesture is to 229 // completion. 230 onInvocationProgress(INVOCATION_TYPE_GESTURE, progress); 231 } 232 233 @Override 234 public void onAssistantGestureCompletion(float velocity) { 235 onGestureCompletion(velocity); 236 } 237 }); 238 } 239 registerVoiceInteractionSessionListener()240 protected void registerVoiceInteractionSessionListener() { 241 mAssistUtils.registerVoiceInteractionSessionListener( 242 new IVoiceInteractionSessionListener.Stub() { 243 @Override 244 public void onVoiceSessionShown() throws RemoteException { 245 if (VERBOSE) { 246 Log.v(TAG, "Voice open"); 247 } 248 mAssistLogger.reportAssistantSessionEvent( 249 AssistantSessionEvent.ASSISTANT_SESSION_UPDATE); 250 } 251 252 @Override 253 public void onVoiceSessionHidden() throws RemoteException { 254 if (VERBOSE) { 255 Log.v(TAG, "Voice closed"); 256 } 257 mAssistLogger.reportAssistantSessionEvent( 258 AssistantSessionEvent.ASSISTANT_SESSION_CLOSE); 259 } 260 261 @Override 262 public void onVoiceSessionWindowVisibilityChanged(boolean visible) 263 throws RemoteException { 264 if (VERBOSE) { 265 Log.v(TAG, "Window visibility changed: " + visible); 266 } 267 } 268 269 @Override 270 public void onSetUiHints(Bundle hints) { 271 if (VERBOSE) { 272 Log.v(TAG, "UI hints received"); 273 } 274 275 String action = hints.getString(ACTION_KEY); 276 if (SET_ASSIST_GESTURE_CONSTRAINED_ACTION.equals(action)) { 277 mSysUiState.get() 278 .setFlag( 279 SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED, 280 hints.getBoolean(CONSTRAINED_KEY, false)) 281 .commitUpdate(mDisplayTracker.getDefaultDisplayId()); 282 } 283 } 284 }); 285 } 286 startAssist(Bundle args)287 public void startAssist(Bundle args) { 288 if (mActivityManager.getLockTaskModeState() == ActivityManager.LOCK_TASK_MODE_LOCKED) { 289 return; 290 } 291 if (shouldOverrideAssist(args)) { 292 try { 293 if (mLauncherProxyService.getProxy() == null) { 294 Log.w(TAG, "No LauncherProxyService to invoke assistant override"); 295 return; 296 } 297 mLauncherProxyService.getProxy().onAssistantOverrideInvoked( 298 args.getInt(INVOCATION_TYPE_KEY)); 299 } catch (RemoteException e) { 300 Log.w(TAG, "Unable to invoke assistant via LauncherProxyService override", e); 301 } 302 return; 303 } 304 305 final ComponentName assistComponent = getAssistInfo(); 306 if (assistComponent == null) { 307 return; 308 } 309 310 final boolean isService = assistComponent.equals(getVoiceInteractorComponentName()); 311 312 if (args == null) { 313 args = new Bundle(); 314 } 315 int legacyInvocationType = args.getInt(INVOCATION_TYPE_KEY, 0); 316 int legacyDeviceState = mPhoneStateMonitor.getPhoneState(); 317 args.putInt(INVOCATION_PHONE_STATE_KEY, legacyDeviceState); 318 args.putLong(INVOCATION_TIME_MS_KEY, SystemClock.elapsedRealtime()); 319 mAssistLogger.reportAssistantInvocationEventFromLegacy( 320 legacyInvocationType, 321 /* isInvocationComplete = */ true, 322 assistComponent, 323 legacyDeviceState); 324 logStartAssistLegacy(legacyInvocationType, legacyDeviceState); 325 mInteractor.onAssistantStarted(legacyInvocationType); 326 startAssistInternal(args, assistComponent, isService); 327 } 328 shouldOverrideAssist(Bundle args)329 private boolean shouldOverrideAssist(Bundle args) { 330 if (args == null || !args.containsKey(INVOCATION_TYPE_KEY)) { 331 return false; 332 } 333 334 int invocationType = args.getInt(INVOCATION_TYPE_KEY); 335 return shouldOverrideAssist(invocationType); 336 } 337 338 /** @return true if the invocation type should be handled by LauncherProxy instead of SysUI. */ shouldOverrideAssist(int invocationType)339 public boolean shouldOverrideAssist(int invocationType) { 340 return mAssistOverrideInvocationTypes != null 341 && Arrays.stream(mAssistOverrideInvocationTypes).anyMatch( 342 override -> override == invocationType); 343 } 344 345 /** 346 * @param invocationTypes The invocation types that will henceforth be handled via 347 * LauncherProxy (Launcher); other invocation types should be handled by 348 * this class. 349 */ setAssistantOverridesRequested(int[] invocationTypes)350 public void setAssistantOverridesRequested(int[] invocationTypes) { 351 mAssistOverrideInvocationTypes = invocationTypes; 352 } 353 354 /** Called when the user is performing an assistant invocation action (e.g. Active Edge) */ onInvocationProgress(int type, float progress)355 public void onInvocationProgress(int type, float progress) { 356 mUiController.onInvocationProgress(type, progress); 357 } 358 359 /** 360 * Called when the user has invoked the assistant with the incoming velocity, in pixels per 361 * millisecond. For invocations without a velocity (e.g. slow drag), the velocity is set to 362 * zero. 363 */ onGestureCompletion(float velocity)364 public void onGestureCompletion(float velocity) { 365 mUiController.onGestureCompletion(velocity); 366 } 367 hideAssist()368 public void hideAssist() { 369 mAssistUtils.hideCurrentSession(); 370 } 371 372 /** 373 * Add the given {@link VisualQueryAttentionListener} to the list of listeners awaiting 374 * notification of gaining/losing visual query attention. 375 */ addVisualQueryAttentionListener(VisualQueryAttentionListener listener)376 public void addVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 377 if (!mVisualQueryAttentionListeners.contains(listener)) { 378 mVisualQueryAttentionListeners.add(listener); 379 } 380 } 381 382 /** 383 * Remove the given {@link VisualQueryAttentionListener} from the list of listeners awaiting 384 * notification of gaining/losing visual query attention. 385 */ removeVisualQueryAttentionListener(VisualQueryAttentionListener listener)386 public void removeVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 387 mVisualQueryAttentionListeners.remove(listener); 388 } 389 startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, boolean isService)390 private void startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, 391 boolean isService) { 392 if (isService) { 393 startVoiceInteractor(args); 394 } else { 395 startAssistActivity(args, assistComponent); 396 } 397 } 398 startAssistActivity(Bundle args, @NonNull ComponentName assistComponent)399 private void startAssistActivity(Bundle args, @NonNull ComponentName assistComponent) { 400 if (!mDeviceProvisionedController.isDeviceProvisioned()) { 401 return; 402 } 403 404 // Close Recent Apps if needed 405 mCommandQueue.animateCollapsePanels( 406 CommandQueue.FLAG_EXCLUDE_SEARCH_PANEL | CommandQueue.FLAG_EXCLUDE_RECENTS_PANEL, 407 false /* force */); 408 409 boolean structureEnabled = mSecureSettings.getIntForUser( 410 Settings.Secure.ASSIST_STRUCTURE_ENABLED, 1, UserHandle.USER_CURRENT) != 0; 411 412 final SearchManager searchManager = 413 (SearchManager) mContext.getSystemService(Context.SEARCH_SERVICE); 414 if (searchManager == null) { 415 return; 416 } 417 final Intent intent = searchManager.getAssistIntent(structureEnabled); 418 if (intent == null) { 419 return; 420 } 421 intent.setComponent(assistComponent); 422 intent.putExtras(args); 423 424 if (structureEnabled && AssistUtils.isDisclosureEnabled(mContext)) { 425 showDisclosure(); 426 } 427 428 try { 429 final ActivityOptions opts = ActivityOptions.makeCustomAnimation(mContext, 430 R.anim.search_launch_enter, R.anim.search_launch_exit); 431 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 432 AsyncTask.execute(new Runnable() { 433 @Override 434 public void run() { 435 mContext.startActivityAsUser(intent, opts.toBundle(), 436 mUserTracker.getUserHandle()); 437 } 438 }); 439 } catch (ActivityNotFoundException e) { 440 Log.w(TAG, "Activity not found for " + intent.getAction()); 441 } 442 } 443 startVoiceInteractor(Bundle args)444 private void startVoiceInteractor(Bundle args) { 445 mAssistUtils.showSessionForActiveService(args, 446 VoiceInteractionSession.SHOW_SOURCE_ASSIST_GESTURE, mContext.getAttributionTag(), 447 null, null); 448 } 449 registerVisualQueryRecognitionStatusListener()450 private void registerVisualQueryRecognitionStatusListener() { 451 if (!mContext.getResources() 452 .getBoolean(R.bool.config_enableVisualQueryAttentionDetection)) { 453 return; 454 } 455 456 mAssistUtils.subscribeVisualQueryRecognitionStatus( 457 new IVisualQueryRecognitionStatusListener.Stub() { 458 @Override 459 public void onStartPerceiving() { 460 mAssistUtils.enableVisualQueryDetection( 461 mVisualQueryDetectionAttentionListener); 462 final StatusBarManager statusBarManager = 463 mContext.getSystemService(StatusBarManager.class); 464 if (statusBarManager != null) { 465 statusBarManager.setIcon("assist_attention", 466 R.drawable.ic_assistant_attention_indicator, 467 0, "Attention Icon for Assistant"); 468 statusBarManager.setIconVisibility("assist_attention", false); 469 } 470 } 471 472 @Override 473 public void onStopPerceiving() { 474 // Treat this as a signal that attention has been lost (and inform listeners 475 // accordingly). 476 handleVisualAttentionChanged(false); 477 mAssistUtils.disableVisualQueryDetection(); 478 final StatusBarManager statusBarManager = 479 mContext.getSystemService(StatusBarManager.class); 480 if (statusBarManager != null) { 481 statusBarManager.removeIcon("assist_attention"); 482 } 483 } 484 }); 485 } 486 487 // TODO (b/319132184): Implemented this with different types. handleVisualAttentionChanged(boolean attentionGained)488 private void handleVisualAttentionChanged(boolean attentionGained) { 489 final StatusBarManager statusBarManager = mContext.getSystemService(StatusBarManager.class); 490 if (statusBarManager != null) { 491 statusBarManager.setIconVisibility("assist_attention", attentionGained); 492 } 493 mVisualQueryAttentionListeners.forEach( 494 attentionGained 495 ? VisualQueryAttentionListener::onAttentionGained 496 : VisualQueryAttentionListener::onAttentionLost); 497 } 498 launchVoiceAssistFromKeyguard()499 public void launchVoiceAssistFromKeyguard() { 500 mAssistUtils.launchVoiceAssistFromKeyguard(); 501 } 502 canVoiceAssistBeLaunchedFromKeyguard()503 public boolean canVoiceAssistBeLaunchedFromKeyguard() { 504 // TODO(b/140051519) 505 return whitelistIpcs(() -> mAssistUtils.activeServiceSupportsLaunchFromKeyguard()); 506 } 507 getVoiceInteractorComponentName()508 public ComponentName getVoiceInteractorComponentName() { 509 return mAssistUtils.getActiveServiceComponentName(); 510 } 511 isVoiceSessionRunning()512 private boolean isVoiceSessionRunning() { 513 return mAssistUtils.isSessionRunning(); 514 } 515 516 @Nullable getAssistInfoForUser(int userId)517 public ComponentName getAssistInfoForUser(int userId) { 518 return mAssistUtils.getAssistComponentForUser(userId); 519 } 520 521 @Nullable getAssistInfo()522 public ComponentName getAssistInfo() { 523 return getAssistInfoForUser(mSelectedUserInteractor.getSelectedUserId()); 524 } 525 showDisclosure()526 public void showDisclosure() { 527 mAssistDisclosure.postShow(); 528 } 529 onLockscreenShown()530 public void onLockscreenShown() { 531 AsyncTask.execute(new Runnable() { 532 @Override 533 public void run() { 534 mAssistUtils.onLockscreenShown(); 535 } 536 }); 537 } 538 539 /** Returns the logging flags for the given Assistant invocation type. */ toLoggingSubType(int invocationType)540 public int toLoggingSubType(int invocationType) { 541 return toLoggingSubType(invocationType, mPhoneStateMonitor.getPhoneState()); 542 } 543 logStartAssistLegacy(int invocationType, int phoneState)544 protected void logStartAssistLegacy(int invocationType, int phoneState) { 545 MetricsLogger.action( 546 new LogMaker(MetricsEvent.ASSISTANT) 547 .setType(MetricsEvent.TYPE_OPEN) 548 .setSubtype(toLoggingSubType(invocationType, phoneState))); 549 } 550 toLoggingSubType(int invocationType, int phoneState)551 protected final int toLoggingSubType(int invocationType, int phoneState) { 552 // Note that this logic will break if the number of Assistant invocation types exceeds 7. 553 // There are currently 5 invocation types, but we will be migrating to the new logging 554 // framework in the next update. 555 int subType = 0; 556 subType |= invocationType << 1; 557 subType |= phoneState << 4; 558 return subType; 559 } 560 } 561