1 package com.android.systemui.assist; 2 3 import static com.android.systemui.DejankUtils.whitelistIpcs; 4 import static com.android.systemui.shared.system.QuickStepContract.SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED; 5 6 import android.annotation.NonNull; 7 import android.annotation.Nullable; 8 import android.app.ActivityOptions; 9 import android.app.SearchManager; 10 import android.content.ActivityNotFoundException; 11 import android.content.ComponentName; 12 import android.content.Context; 13 import android.content.Intent; 14 import android.metrics.LogMaker; 15 import android.os.AsyncTask; 16 import android.os.Bundle; 17 import android.os.Handler; 18 import android.os.RemoteException; 19 import android.os.SystemClock; 20 import android.os.UserHandle; 21 import android.provider.Settings; 22 import android.service.voice.VoiceInteractionSession; 23 import android.util.Log; 24 25 import com.android.internal.app.AssistUtils; 26 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 27 import com.android.internal.app.IVisualQueryRecognitionStatusListener; 28 import com.android.internal.app.IVoiceInteractionSessionListener; 29 import com.android.internal.logging.MetricsLogger; 30 import com.android.internal.logging.nano.MetricsProto.MetricsEvent; 31 import com.android.keyguard.KeyguardUpdateMonitor; 32 import com.android.systemui.R; 33 import com.android.systemui.assist.ui.DefaultUiController; 34 import com.android.systemui.dagger.SysUISingleton; 35 import com.android.systemui.dagger.qualifiers.Main; 36 import com.android.systemui.model.SysUiState; 37 import com.android.systemui.recents.OverviewProxyService; 38 import com.android.systemui.settings.DisplayTracker; 39 import com.android.systemui.settings.UserTracker; 40 import com.android.systemui.statusbar.CommandQueue; 41 import com.android.systemui.statusbar.policy.DeviceProvisionedController; 42 import com.android.systemui.util.settings.SecureSettings; 43 44 import dagger.Lazy; 45 46 import java.util.ArrayList; 47 import java.util.Arrays; 48 import java.util.List; 49 50 import javax.inject.Inject; 51 52 /** 53 * Class to manage everything related to assist in SystemUI. 54 */ 55 @SysUISingleton 56 public class AssistManager { 57 58 /** 59 * Controls the UI for showing Assistant invocation progress. 60 */ 61 public interface UiController { 62 /** 63 * Updates the invocation progress. 64 * 65 * @param type one of INVOCATION_TYPE_GESTURE, INVOCATION_TYPE_ACTIVE_EDGE, 66 * INVOCATION_TYPE_VOICE, INVOCATION_TYPE_QUICK_SEARCH_BAR, 67 * INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS 68 * @param progress a float between 0 and 1 inclusive. 0 represents the beginning of the 69 * gesture; 1 represents the end. 70 */ onInvocationProgress(int type, float progress)71 void onInvocationProgress(int type, float progress); 72 73 /** 74 * Called when an invocation gesture completes. 75 * 76 * @param velocity the speed of the invocation gesture, in pixels per millisecond. For 77 * drags, this is 0. 78 */ onGestureCompletion(float velocity)79 void onGestureCompletion(float velocity); 80 81 /** 82 * Hides any SysUI for the assistant, but _does not_ close the assistant itself. 83 */ hide()84 void hide(); 85 } 86 87 /** 88 * An interface for a listener that receives notification that visual query attention has 89 * either been gained or lost. 90 */ 91 public interface VisualQueryAttentionListener { 92 /** Called when visual query attention has been gained. */ onAttentionGained()93 void onAttentionGained(); 94 95 /** Called when visual query attention has been lost. */ onAttentionLost()96 void onAttentionLost(); 97 } 98 99 private static final String TAG = "AssistManager"; 100 101 // Note that VERBOSE logging may leak PII (e.g. transcription contents). 102 private static final boolean VERBOSE = false; 103 104 private static final String INVOCATION_TIME_MS_KEY = "invocation_time_ms"; 105 private static final String INVOCATION_PHONE_STATE_KEY = "invocation_phone_state"; 106 protected static final String ACTION_KEY = "action"; 107 protected static final String SET_ASSIST_GESTURE_CONSTRAINED_ACTION = 108 "set_assist_gesture_constrained"; 109 protected static final String CONSTRAINED_KEY = "should_constrain"; 110 111 public static final String INVOCATION_TYPE_KEY = "invocation_type"; 112 public static final int INVOCATION_TYPE_UNKNOWN = 113 AssistUtils.INVOCATION_TYPE_UNKNOWN; 114 public static final int INVOCATION_TYPE_GESTURE = 115 AssistUtils.INVOCATION_TYPE_GESTURE; 116 public static final int INVOCATION_TYPE_OTHER = 117 AssistUtils.INVOCATION_TYPE_PHYSICAL_GESTURE; 118 public static final int INVOCATION_TYPE_VOICE = 119 AssistUtils.INVOCATION_TYPE_VOICE; 120 public static final int INVOCATION_TYPE_QUICK_SEARCH_BAR = 121 AssistUtils.INVOCATION_TYPE_QUICK_SEARCH_BAR; 122 public static final int INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS = 123 AssistUtils.INVOCATION_TYPE_HOME_BUTTON_LONG_PRESS; 124 public static final int INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS = 125 AssistUtils.INVOCATION_TYPE_POWER_BUTTON_LONG_PRESS; 126 public static final int INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS = 127 AssistUtils.INVOCATION_TYPE_NAV_HANDLE_LONG_PRESS; 128 129 public static final int DISMISS_REASON_INVOCATION_CANCELLED = 1; 130 public static final int DISMISS_REASON_TAP = 2; 131 public static final int DISMISS_REASON_BACK = 3; 132 public static final int DISMISS_REASON_TIMEOUT = 4; 133 134 private static final long TIMEOUT_SERVICE = 2500; 135 private static final long TIMEOUT_ACTIVITY = 1000; 136 137 protected final Context mContext; 138 private final AssistDisclosure mAssistDisclosure; 139 private final PhoneStateMonitor mPhoneStateMonitor; 140 private final OverviewProxyService mOverviewProxyService; 141 private final UiController mUiController; 142 protected final Lazy<SysUiState> mSysUiState; 143 protected final AssistLogger mAssistLogger; 144 private final UserTracker mUserTracker; 145 private final DisplayTracker mDisplayTracker; 146 private final SecureSettings mSecureSettings; 147 148 private final DeviceProvisionedController mDeviceProvisionedController; 149 150 private final List<VisualQueryAttentionListener> mVisualQueryAttentionListeners = 151 new ArrayList<>(); 152 153 private final IVisualQueryDetectionAttentionListener mVisualQueryDetectionAttentionListener = 154 new IVisualQueryDetectionAttentionListener.Stub() { 155 @Override 156 public void onAttentionGained() { 157 handleVisualAttentionChanged(true); 158 } 159 160 @Override 161 public void onAttentionLost() { 162 handleVisualAttentionChanged(false); 163 } 164 }; 165 166 private final CommandQueue mCommandQueue; 167 protected final AssistUtils mAssistUtils; 168 169 // Invocation types that should be sent over OverviewProxy instead of handled here. 170 private int[] mAssistOverrideInvocationTypes; 171 172 @Inject AssistManager( DeviceProvisionedController controller, Context context, AssistUtils assistUtils, CommandQueue commandQueue, PhoneStateMonitor phoneStateMonitor, OverviewProxyService overviewProxyService, Lazy<SysUiState> sysUiState, DefaultUiController defaultUiController, AssistLogger assistLogger, @Main Handler uiHandler, UserTracker userTracker, DisplayTracker displayTracker, SecureSettings secureSettings)173 public AssistManager( 174 DeviceProvisionedController controller, 175 Context context, 176 AssistUtils assistUtils, 177 CommandQueue commandQueue, 178 PhoneStateMonitor phoneStateMonitor, 179 OverviewProxyService overviewProxyService, 180 Lazy<SysUiState> sysUiState, 181 DefaultUiController defaultUiController, 182 AssistLogger assistLogger, 183 @Main Handler uiHandler, 184 UserTracker userTracker, 185 DisplayTracker displayTracker, 186 SecureSettings secureSettings) { 187 mContext = context; 188 mDeviceProvisionedController = controller; 189 mCommandQueue = commandQueue; 190 mAssistUtils = assistUtils; 191 mAssistDisclosure = new AssistDisclosure(context, uiHandler); 192 mOverviewProxyService = overviewProxyService; 193 mPhoneStateMonitor = phoneStateMonitor; 194 mAssistLogger = assistLogger; 195 mUserTracker = userTracker; 196 mDisplayTracker = displayTracker; 197 mSecureSettings = secureSettings; 198 199 registerVoiceInteractionSessionListener(); 200 registerVisualQueryRecognitionStatusListener(); 201 202 mUiController = defaultUiController; 203 204 mSysUiState = sysUiState; 205 206 mOverviewProxyService.addCallback(new OverviewProxyService.OverviewProxyListener() { 207 @Override 208 public void onAssistantProgress(float progress) { 209 // Progress goes from 0 to 1 to indicate how close the assist gesture is to 210 // completion. 211 onInvocationProgress(INVOCATION_TYPE_GESTURE, progress); 212 } 213 214 @Override 215 public void onAssistantGestureCompletion(float velocity) { 216 onGestureCompletion(velocity); 217 } 218 }); 219 } 220 registerVoiceInteractionSessionListener()221 protected void registerVoiceInteractionSessionListener() { 222 mAssistUtils.registerVoiceInteractionSessionListener( 223 new IVoiceInteractionSessionListener.Stub() { 224 @Override 225 public void onVoiceSessionShown() throws RemoteException { 226 if (VERBOSE) { 227 Log.v(TAG, "Voice open"); 228 } 229 mAssistLogger.reportAssistantSessionEvent( 230 AssistantSessionEvent.ASSISTANT_SESSION_UPDATE); 231 } 232 233 @Override 234 public void onVoiceSessionHidden() throws RemoteException { 235 if (VERBOSE) { 236 Log.v(TAG, "Voice closed"); 237 } 238 mAssistLogger.reportAssistantSessionEvent( 239 AssistantSessionEvent.ASSISTANT_SESSION_CLOSE); 240 } 241 242 @Override 243 public void onVoiceSessionWindowVisibilityChanged(boolean visible) 244 throws RemoteException { 245 if (VERBOSE) { 246 Log.v(TAG, "Window visibility changed: " + visible); 247 } 248 } 249 250 @Override 251 public void onSetUiHints(Bundle hints) { 252 if (VERBOSE) { 253 Log.v(TAG, "UI hints received"); 254 } 255 256 String action = hints.getString(ACTION_KEY); 257 if (SET_ASSIST_GESTURE_CONSTRAINED_ACTION.equals(action)) { 258 mSysUiState.get() 259 .setFlag( 260 SYSUI_STATE_ASSIST_GESTURE_CONSTRAINED, 261 hints.getBoolean(CONSTRAINED_KEY, false)) 262 .commitUpdate(mDisplayTracker.getDefaultDisplayId()); 263 } 264 } 265 }); 266 } 267 startAssist(Bundle args)268 public void startAssist(Bundle args) { 269 if (shouldOverrideAssist(args)) { 270 try { 271 if (mOverviewProxyService.getProxy() == null) { 272 Log.w(TAG, "No OverviewProxyService to invoke assistant override"); 273 return; 274 } 275 mOverviewProxyService.getProxy().onAssistantOverrideInvoked( 276 args.getInt(INVOCATION_TYPE_KEY)); 277 } catch (RemoteException e) { 278 Log.w(TAG, "Unable to invoke assistant via OverviewProxyService override", e); 279 } 280 return; 281 } 282 283 final ComponentName assistComponent = getAssistInfo(); 284 if (assistComponent == null) { 285 return; 286 } 287 288 final boolean isService = assistComponent.equals(getVoiceInteractorComponentName()); 289 290 if (args == null) { 291 args = new Bundle(); 292 } 293 int legacyInvocationType = args.getInt(INVOCATION_TYPE_KEY, 0); 294 int legacyDeviceState = mPhoneStateMonitor.getPhoneState(); 295 args.putInt(INVOCATION_PHONE_STATE_KEY, legacyDeviceState); 296 args.putLong(INVOCATION_TIME_MS_KEY, SystemClock.elapsedRealtime()); 297 mAssistLogger.reportAssistantInvocationEventFromLegacy( 298 legacyInvocationType, 299 /* isInvocationComplete = */ true, 300 assistComponent, 301 legacyDeviceState); 302 logStartAssistLegacy(legacyInvocationType, legacyDeviceState); 303 startAssistInternal(args, assistComponent, isService); 304 } 305 shouldOverrideAssist(Bundle args)306 private boolean shouldOverrideAssist(Bundle args) { 307 if (args == null || !args.containsKey(INVOCATION_TYPE_KEY)) { 308 return false; 309 } 310 311 int invocationType = args.getInt(INVOCATION_TYPE_KEY); 312 return shouldOverrideAssist(invocationType); 313 } 314 315 /** @return true if the invocation type should be handled by OverviewProxy instead of SysUI. */ shouldOverrideAssist(int invocationType)316 public boolean shouldOverrideAssist(int invocationType) { 317 return mAssistOverrideInvocationTypes != null 318 && Arrays.stream(mAssistOverrideInvocationTypes).anyMatch( 319 override -> override == invocationType); 320 } 321 322 /** 323 * @param invocationTypes The invocation types that will henceforth be handled via 324 * OverviewProxy (Launcher); other invocation types should be handled by this class. 325 */ setAssistantOverridesRequested(int[] invocationTypes)326 public void setAssistantOverridesRequested(int[] invocationTypes) { 327 mAssistOverrideInvocationTypes = invocationTypes; 328 } 329 330 /** Called when the user is performing an assistant invocation action (e.g. Active Edge) */ onInvocationProgress(int type, float progress)331 public void onInvocationProgress(int type, float progress) { 332 mUiController.onInvocationProgress(type, progress); 333 } 334 335 /** 336 * Called when the user has invoked the assistant with the incoming velocity, in pixels per 337 * millisecond. For invocations without a velocity (e.g. slow drag), the velocity is set to 338 * zero. 339 */ onGestureCompletion(float velocity)340 public void onGestureCompletion(float velocity) { 341 mUiController.onGestureCompletion(velocity); 342 } 343 hideAssist()344 public void hideAssist() { 345 mAssistUtils.hideCurrentSession(); 346 } 347 348 /** 349 * Add the given {@link VisualQueryAttentionListener} to the list of listeners awaiting 350 * notification of gaining/losing visual query attention. 351 */ addVisualQueryAttentionListener(VisualQueryAttentionListener listener)352 public void addVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 353 if (!mVisualQueryAttentionListeners.contains(listener)) { 354 mVisualQueryAttentionListeners.add(listener); 355 } 356 } 357 358 /** 359 * Remove the given {@link VisualQueryAttentionListener} from the list of listeners awaiting 360 * notification of gaining/losing visual query attention. 361 */ removeVisualQueryAttentionListener(VisualQueryAttentionListener listener)362 public void removeVisualQueryAttentionListener(VisualQueryAttentionListener listener) { 363 mVisualQueryAttentionListeners.remove(listener); 364 } 365 startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, boolean isService)366 private void startAssistInternal(Bundle args, @NonNull ComponentName assistComponent, 367 boolean isService) { 368 if (isService) { 369 startVoiceInteractor(args); 370 } else { 371 startAssistActivity(args, assistComponent); 372 } 373 } 374 startAssistActivity(Bundle args, @NonNull ComponentName assistComponent)375 private void startAssistActivity(Bundle args, @NonNull ComponentName assistComponent) { 376 if (!mDeviceProvisionedController.isDeviceProvisioned()) { 377 return; 378 } 379 380 // Close Recent Apps if needed 381 mCommandQueue.animateCollapsePanels( 382 CommandQueue.FLAG_EXCLUDE_SEARCH_PANEL | CommandQueue.FLAG_EXCLUDE_RECENTS_PANEL, 383 false /* force */); 384 385 boolean structureEnabled = mSecureSettings.getIntForUser( 386 Settings.Secure.ASSIST_STRUCTURE_ENABLED, 1, UserHandle.USER_CURRENT) != 0; 387 388 final SearchManager searchManager = 389 (SearchManager) mContext.getSystemService(Context.SEARCH_SERVICE); 390 if (searchManager == null) { 391 return; 392 } 393 final Intent intent = searchManager.getAssistIntent(structureEnabled); 394 if (intent == null) { 395 return; 396 } 397 intent.setComponent(assistComponent); 398 intent.putExtras(args); 399 400 if (structureEnabled && AssistUtils.isDisclosureEnabled(mContext)) { 401 showDisclosure(); 402 } 403 404 try { 405 final ActivityOptions opts = ActivityOptions.makeCustomAnimation(mContext, 406 R.anim.search_launch_enter, R.anim.search_launch_exit); 407 intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 408 AsyncTask.execute(new Runnable() { 409 @Override 410 public void run() { 411 mContext.startActivityAsUser(intent, opts.toBundle(), 412 mUserTracker.getUserHandle()); 413 } 414 }); 415 } catch (ActivityNotFoundException e) { 416 Log.w(TAG, "Activity not found for " + intent.getAction()); 417 } 418 } 419 startVoiceInteractor(Bundle args)420 private void startVoiceInteractor(Bundle args) { 421 mAssistUtils.showSessionForActiveService(args, 422 VoiceInteractionSession.SHOW_SOURCE_ASSIST_GESTURE, mContext.getAttributionTag(), 423 null, null); 424 } 425 registerVisualQueryRecognitionStatusListener()426 private void registerVisualQueryRecognitionStatusListener() { 427 if (!mContext.getResources() 428 .getBoolean(R.bool.config_enableVisualQueryAttentionDetection)) { 429 return; 430 } 431 432 mAssistUtils.subscribeVisualQueryRecognitionStatus( 433 new IVisualQueryRecognitionStatusListener.Stub() { 434 @Override 435 public void onStartPerceiving() { 436 mAssistUtils.enableVisualQueryDetection( 437 mVisualQueryDetectionAttentionListener); 438 } 439 440 @Override 441 public void onStopPerceiving() { 442 // Treat this as a signal that attention has been lost (and inform listeners 443 // accordingly). 444 handleVisualAttentionChanged(false); 445 mAssistUtils.disableVisualQueryDetection(); 446 } 447 }); 448 } 449 handleVisualAttentionChanged(boolean attentionGained)450 private void handleVisualAttentionChanged(boolean attentionGained) { 451 mVisualQueryAttentionListeners.forEach( 452 attentionGained 453 ? VisualQueryAttentionListener::onAttentionGained 454 : VisualQueryAttentionListener::onAttentionLost); 455 } 456 launchVoiceAssistFromKeyguard()457 public void launchVoiceAssistFromKeyguard() { 458 mAssistUtils.launchVoiceAssistFromKeyguard(); 459 } 460 canVoiceAssistBeLaunchedFromKeyguard()461 public boolean canVoiceAssistBeLaunchedFromKeyguard() { 462 // TODO(b/140051519) 463 return whitelistIpcs(() -> mAssistUtils.activeServiceSupportsLaunchFromKeyguard()); 464 } 465 getVoiceInteractorComponentName()466 public ComponentName getVoiceInteractorComponentName() { 467 return mAssistUtils.getActiveServiceComponentName(); 468 } 469 isVoiceSessionRunning()470 private boolean isVoiceSessionRunning() { 471 return mAssistUtils.isSessionRunning(); 472 } 473 474 @Nullable getAssistInfoForUser(int userId)475 public ComponentName getAssistInfoForUser(int userId) { 476 return mAssistUtils.getAssistComponentForUser(userId); 477 } 478 479 @Nullable getAssistInfo()480 private ComponentName getAssistInfo() { 481 return getAssistInfoForUser(KeyguardUpdateMonitor.getCurrentUser()); 482 } 483 showDisclosure()484 public void showDisclosure() { 485 mAssistDisclosure.postShow(); 486 } 487 onLockscreenShown()488 public void onLockscreenShown() { 489 AsyncTask.execute(new Runnable() { 490 @Override 491 public void run() { 492 mAssistUtils.onLockscreenShown(); 493 } 494 }); 495 } 496 497 /** Returns the logging flags for the given Assistant invocation type. */ toLoggingSubType(int invocationType)498 public int toLoggingSubType(int invocationType) { 499 return toLoggingSubType(invocationType, mPhoneStateMonitor.getPhoneState()); 500 } 501 logStartAssistLegacy(int invocationType, int phoneState)502 protected void logStartAssistLegacy(int invocationType, int phoneState) { 503 MetricsLogger.action( 504 new LogMaker(MetricsEvent.ASSISTANT) 505 .setType(MetricsEvent.TYPE_OPEN) 506 .setSubtype(toLoggingSubType(invocationType, phoneState))); 507 } 508 toLoggingSubType(int invocationType, int phoneState)509 protected final int toLoggingSubType(int invocationType, int phoneState) { 510 // Note that this logic will break if the number of Assistant invocation types exceeds 7. 511 // There are currently 5 invocation types, but we will be migrating to the new logging 512 // framework in the next update. 513 int subType = 0; 514 subType |= invocationType << 1; 515 subType |= phoneState << 4; 516 return subType; 517 } 518 } 519