1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED; 20 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE; 21 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED; 22 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED; 23 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE; 24 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL; 25 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR; 26 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP; 27 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER; 28 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH; 29 30 import android.annotation.NonNull; 31 import android.annotation.Nullable; 32 import android.app.AppOpsManager; 33 import android.compat.annotation.ChangeId; 34 import android.compat.annotation.Disabled; 35 import android.content.ComponentName; 36 import android.content.ContentCaptureOptions; 37 import android.content.Context; 38 import android.content.Intent; 39 import android.hardware.soundtrigger.IRecognitionStatusCallback; 40 import android.hardware.soundtrigger.SoundTrigger; 41 import android.media.AudioFormat; 42 import android.media.AudioManagerInternal; 43 import android.media.permission.Identity; 44 import android.os.Binder; 45 import android.os.Bundle; 46 import android.os.Handler; 47 import android.os.HandlerThread; 48 import android.os.IBinder; 49 import android.os.IRemoteCallback; 50 import android.os.ParcelFileDescriptor; 51 import android.os.PersistableBundle; 52 import android.os.RemoteException; 53 import android.os.ServiceManager; 54 import android.os.SharedMemory; 55 import android.os.SystemProperties; 56 import android.provider.DeviceConfig; 57 import android.service.voice.HotwordDetectionService; 58 import android.service.voice.HotwordDetectionServiceFailure; 59 import android.service.voice.HotwordDetector; 60 import android.service.voice.IDetectorSessionStorageService; 61 import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; 62 import android.service.voice.ISandboxedDetectionService; 63 import android.service.voice.IVisualQueryDetectionVoiceInteractionCallback; 64 import android.service.voice.SoundTriggerFailure; 65 import android.service.voice.VisualQueryDetectionService; 66 import android.service.voice.VisualQueryDetectionServiceFailure; 67 import android.service.voice.VoiceInteractionManagerInternal.HotwordDetectionServiceIdentity; 68 import android.service.voice.VoiceInteractionManagerInternal.WearableHotwordDetectionCallback; 69 import android.speech.IRecognitionServiceManager; 70 import android.util.Slog; 71 import android.util.SparseArray; 72 import android.view.contentcapture.IContentCaptureManager; 73 74 import com.android.internal.annotations.GuardedBy; 75 import com.android.internal.app.IHotwordRecognitionStatusCallback; 76 import com.android.internal.app.IVisualQueryDetectionAttentionListener; 77 import com.android.internal.app.IVoiceInteractionAccessibilitySettingsListener; 78 import com.android.internal.infra.AndroidFuture; 79 import com.android.internal.infra.ServiceConnector; 80 import com.android.server.LocalServices; 81 import com.android.server.pm.permission.PermissionManagerServiceInternal; 82 import com.android.server.voiceinteraction.VoiceInteractionManagerServiceImpl.DetectorRemoteExceptionListener; 83 84 import java.io.PrintWriter; 85 import java.time.Instant; 86 import java.util.concurrent.ScheduledFuture; 87 import java.util.concurrent.ScheduledThreadPoolExecutor; 88 import java.util.concurrent.TimeUnit; 89 import java.util.function.Consumer; 90 import java.util.function.Function; 91 92 /** 93 * A class that provides the communication with the {@link HotwordDetectionService} and 94 * {@link VisualQueryDetectionService}. 95 */ 96 final class HotwordDetectionConnection { 97 private static final String TAG = "HotwordDetectionConnection"; 98 static final boolean DEBUG = false; 99 100 /** 101 * Implementors of the HotwordDetectionService must not augment the phrase IDs which are 102 * supplied via HotwordDetectionService 103 * #onDetect(AlwaysOnHotwordDetector.EventPayload, long, HotwordDetectionService.Callback). 104 * 105 * <p>The HotwordDetectedResult#getHotwordPhraseId() must match one of the phrase IDs 106 * from the AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras() list. 107 * </p> 108 * 109 * <p>This behavior change is made to ensure the HotwordDetectionService honors what 110 * it receives from the android.hardware.soundtrigger.SoundTriggerModule, and it 111 * cannot signal to the client application a phrase which was not originally detected. 112 * </p> 113 */ 114 @ChangeId 115 @Disabled 116 public static final long ENFORCE_HOTWORD_PHRASE_ID = 215066299L; 117 118 private static final String KEY_RESTART_PERIOD_IN_SECONDS = "restart_period_in_seconds"; 119 private static final long RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS = 60 * 60 * 1000; // 1 hour 120 private static final int MAX_ISOLATED_PROCESS_NUMBER = 10; 121 122 private static final boolean SYSPROP_VISUAL_QUERY_SERVICE_ENABLED = 123 SystemProperties.getBoolean("ro.hotword.visual_query_service_enabled", false); 124 125 /** 126 * Indicates the {@link HotwordDetectionService} is created. 127 */ 128 private static final int DETECTION_SERVICE_TYPE_HOTWORD = 1; 129 130 /** 131 * Indicates the {@link VisualQueryDetectionService} is created. 132 */ 133 private static final int DETECTION_SERVICE_TYPE_VISUAL_QUERY = 2; 134 135 // TODO: This may need to be a Handler(looper) 136 private final ScheduledThreadPoolExecutor mScheduledExecutorService = 137 new ScheduledThreadPoolExecutor(1); 138 @Nullable private final ScheduledFuture<?> mCancellationTaskFuture; 139 private final IBinder.DeathRecipient mAudioServerDeathRecipient = this::audioServerDied; 140 @NonNull private final ServiceConnectionFactory mHotwordDetectionServiceConnectionFactory; 141 @NonNull private final ServiceConnectionFactory mVisualQueryDetectionServiceConnectionFactory; 142 private int mDetectorType; 143 /** 144 * Time after which each HotwordDetectionService process is stopped and replaced by a new one. 145 * 0 indicates no restarts. 146 */ 147 private final int mReStartPeriodSeconds; 148 149 final Object mLock; 150 final int mVoiceInteractionServiceUid; 151 final ComponentName mHotwordDetectionComponentName; 152 final ComponentName mVisualQueryDetectionComponentName; 153 final int mUserId; 154 final Context mContext; 155 final AccessibilitySettingsListener mAccessibilitySettingsListener; 156 volatile HotwordDetectionServiceIdentity mIdentity; 157 //TODO: Consider rename this to SandboxedDetectionIdentity 158 private Instant mLastRestartInstant; 159 160 private ScheduledFuture<?> mDebugHotwordLoggingTimeoutFuture = null; 161 162 /** Identity used for attributing app ops when delivering data to the Interactor. */ 163 @GuardedBy("mLock") 164 @Nullable 165 private final Identity mVoiceInteractorIdentity; 166 private int mRestartCount = 0; 167 @NonNull private ServiceConnection mRemoteHotwordDetectionService; 168 @NonNull private ServiceConnection mRemoteVisualQueryDetectionService; 169 @GuardedBy("mLock") 170 @Nullable private IBinder mAudioFlinger; 171 172 @Nullable private IHotwordRecognitionStatusCallback mHotwordRecognitionCallback; 173 @GuardedBy("mLock") 174 private boolean mDebugHotwordLogging = false; 175 176 private DetectorRemoteExceptionListener mRemoteExceptionListener; 177 178 /** 179 * For multiple detectors feature, we only support one AlwaysOnHotwordDetector and one 180 * SoftwareHotwordDetector at the same time. We use SparseArray with detector type as the key 181 * to record the detectors. 182 */ 183 @GuardedBy("mLock") 184 private final SparseArray<DetectorSession> mDetectorSessions = 185 new SparseArray<>(); 186 187 /** Listens to changes to voice activation op. */ 188 private final AppOpsManager.OnOpChangedListener mOnOpChangedListener = 189 new AppOpsManager.OnOpChangedListener() { 190 @Override 191 public void onOpChanged(String op, String packageName) { 192 if (op.equals(AppOpsManager.OPSTR_RECEIVE_SANDBOX_TRIGGER_AUDIO)) { 193 AppOpsManager appOpsManager = 194 mContext.getSystemService(AppOpsManager.class); 195 synchronized (mLock) { 196 int checkOp = appOpsManager.unsafeCheckOpNoThrow( 197 AppOpsManager.OPSTR_RECEIVE_SANDBOX_TRIGGER_AUDIO, 198 mVoiceInteractorIdentity.uid, 199 mVoiceInteractorIdentity.packageName); 200 // Voice activation app op disabled, safely shutdown hotword detection. 201 if (checkOp == AppOpsManager.MODE_ERRORED) { 202 Slog.i(TAG, "Shutdown hotword detection service on voice " 203 + "activation op disabled."); 204 safelyShutdownHotwordDetectionOnVoiceActivationDisabledLocked(); 205 } 206 } 207 } 208 } 209 }; 210 211 /** Listen to changes of {@link Settings.Secure.VISUAL_QUERY_ACCESSIBILITY_DETECTION_ENABLED}. 212 * 213 * This is registered to the {@link VoiceInteractionManagerServiceImpl} where all settings 214 * listeners are centralized and notified. 215 */ 216 private final class AccessibilitySettingsListener extends 217 IVoiceInteractionAccessibilitySettingsListener.Stub { 218 @Override onAccessibilityDetectionChanged(boolean enable)219 public void onAccessibilityDetectionChanged(boolean enable) { 220 synchronized (mLock) { 221 if (DEBUG) { 222 Slog.d(TAG, "Update settings change: " + enable); 223 } 224 VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked(); 225 if (session != null) { 226 session.updateAccessibilityEgressStateLocked(enable); 227 } 228 } 229 } 230 } 231 232 HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, ComponentName visualQueryDetectionServiceName, int userId, boolean bindInstantServiceAllowed, int detectorType, DetectorRemoteExceptionListener listener)233 HotwordDetectionConnection(Object lock, Context context, int voiceInteractionServiceUid, 234 Identity voiceInteractorIdentity, ComponentName hotwordDetectionServiceName, 235 ComponentName visualQueryDetectionServiceName, int userId, 236 boolean bindInstantServiceAllowed, int detectorType, 237 DetectorRemoteExceptionListener listener) { 238 mLock = lock; 239 mContext = context; 240 mVoiceInteractionServiceUid = voiceInteractionServiceUid; 241 mVoiceInteractorIdentity = voiceInteractorIdentity; 242 mHotwordDetectionComponentName = hotwordDetectionServiceName; 243 mVisualQueryDetectionComponentName = visualQueryDetectionServiceName; 244 mUserId = userId; 245 mDetectorType = detectorType; 246 mRemoteExceptionListener = listener; 247 mReStartPeriodSeconds = DeviceConfig.getInt(DeviceConfig.NAMESPACE_VOICE_INTERACTION, 248 KEY_RESTART_PERIOD_IN_SECONDS, 0); 249 mAccessibilitySettingsListener = new AccessibilitySettingsListener(); 250 251 final Intent hotwordDetectionServiceIntent = 252 new Intent(HotwordDetectionService.SERVICE_INTERFACE); 253 hotwordDetectionServiceIntent.setComponent(mHotwordDetectionComponentName); 254 255 final Intent visualQueryDetectionServiceIntent = 256 new Intent(VisualQueryDetectionService.SERVICE_INTERFACE); 257 visualQueryDetectionServiceIntent.setComponent(mVisualQueryDetectionComponentName); 258 259 initAudioFlinger(); 260 261 mHotwordDetectionServiceConnectionFactory = 262 new ServiceConnectionFactory(hotwordDetectionServiceIntent, 263 bindInstantServiceAllowed, DETECTION_SERVICE_TYPE_HOTWORD); 264 265 mVisualQueryDetectionServiceConnectionFactory = 266 new ServiceConnectionFactory(visualQueryDetectionServiceIntent, 267 bindInstantServiceAllowed, DETECTION_SERVICE_TYPE_VISUAL_QUERY); 268 269 270 mLastRestartInstant = Instant.now(); 271 272 AppOpsManager appOpsManager = mContext.getSystemService(AppOpsManager.class); 273 appOpsManager.startWatchingMode(AppOpsManager.OP_RECEIVE_SANDBOX_TRIGGER_AUDIO, 274 mVoiceInteractorIdentity.packageName, mOnOpChangedListener); 275 276 if (mReStartPeriodSeconds <= 0) { 277 mCancellationTaskFuture = null; 278 } else { 279 mScheduledExecutorService.setRemoveOnCancelPolicy(true); 280 // TODO: we need to be smarter here, e.g. schedule it a bit more often, 281 // but wait until the current session is closed. 282 mCancellationTaskFuture = mScheduledExecutorService.scheduleAtFixedRate(() -> { 283 Slog.v(TAG, "Time to restart the process, TTL has passed"); 284 synchronized (mLock) { 285 restartProcessLocked(); 286 if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 287 HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, 288 HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__SCHEDULE, 289 mVoiceInteractionServiceUid); 290 } 291 } 292 }, mReStartPeriodSeconds, mReStartPeriodSeconds, TimeUnit.SECONDS); 293 } 294 } 295 initAudioFlinger()296 private void initAudioFlinger() { 297 if (DEBUG) { 298 Slog.d(TAG, "initAudioFlinger"); 299 } 300 final IBinder audioFlinger = ServiceManager.waitForService("media.audio_flinger"); 301 if (audioFlinger == null) { 302 setAudioFlinger(null); 303 throw new IllegalStateException("Service media.audio_flinger wasn't found."); 304 } 305 if (DEBUG) { 306 Slog.d(TAG, "Obtained audio_flinger binder."); 307 } 308 try { 309 audioFlinger.linkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); 310 } catch (RemoteException e) { 311 Slog.w(TAG, "Audio server died before we registered a DeathRecipient; " 312 + "retrying init.", e); 313 initAudioFlinger(); 314 return; 315 } 316 317 setAudioFlinger(audioFlinger); 318 } 319 setAudioFlinger(@ullable IBinder audioFlinger)320 private void setAudioFlinger(@Nullable IBinder audioFlinger) { 321 synchronized (mLock) { 322 mAudioFlinger = audioFlinger; 323 } 324 } 325 audioServerDied()326 private void audioServerDied() { 327 Slog.w(TAG, "Audio server died; restarting the HotwordDetectionService."); 328 // TODO: Check if this needs to be scheduled on a different thread. 329 initAudioFlinger(); 330 synchronized (mLock) { 331 // We restart the process instead of simply sending over the new binder, to avoid race 332 // conditions with audio reading in the service. 333 restartProcessLocked(); 334 if (mDetectorType != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 335 HotwordMetricsLogger.writeServiceRestartEvent(mDetectorType, 336 HOTWORD_DETECTION_SERVICE_RESTARTED__REASON__AUDIO_SERVICE_DIED, 337 mVoiceInteractionServiceUid); 338 } 339 } 340 } 341 342 @SuppressWarnings("GuardedBy") cancelLocked()343 void cancelLocked() { 344 Slog.v(TAG, "cancelLocked"); 345 clearDebugHotwordLoggingTimeoutLocked(); 346 mRemoteExceptionListener = null; 347 runForEachDetectorSessionLocked((session) -> { 348 session.destroyLocked(); 349 }); 350 mDetectorSessions.clear(); 351 mDebugHotwordLogging = false; 352 unbindVisualQueryDetectionService(); 353 unbindHotwordDetectionService(); 354 if (mCancellationTaskFuture != null) { 355 mCancellationTaskFuture.cancel(/* mayInterruptIfRunning= */ true); 356 } 357 if (mAudioFlinger != null) { 358 mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); 359 mAudioFlinger = null; 360 } 361 // Unregister the on op mode changed listener. 362 AppOpsManager appOpsManager = mContext.getSystemService(AppOpsManager.class); 363 appOpsManager.stopWatchingMode(mOnOpChangedListener); 364 } 365 366 @SuppressWarnings("GuardedBy") unbindVisualQueryDetectionService()367 private void unbindVisualQueryDetectionService() { 368 if (mRemoteVisualQueryDetectionService != null) { 369 mRemoteVisualQueryDetectionService.unbind(); 370 mRemoteVisualQueryDetectionService = null; 371 } 372 resetDetectionProcessIdentityIfEmptyLocked(); 373 } 374 375 @SuppressWarnings("GuardedBy") unbindHotwordDetectionService()376 private void unbindHotwordDetectionService() { 377 if (mRemoteHotwordDetectionService != null) { 378 mRemoteHotwordDetectionService.unbind(); 379 mRemoteHotwordDetectionService = null; 380 } 381 resetDetectionProcessIdentityIfEmptyLocked(); 382 } 383 384 // TODO(b/266669849): Clean up SuppressWarnings for calling methods. 385 @GuardedBy("mLock") resetDetectionProcessIdentityIfEmptyLocked()386 private void resetDetectionProcessIdentityIfEmptyLocked() { 387 if (mRemoteHotwordDetectionService == null && mRemoteVisualQueryDetectionService == null) { 388 LocalServices.getService(PermissionManagerServiceInternal.class) 389 .setHotwordDetectionServiceProvider(null); 390 if (mIdentity != null) { 391 removeServiceUidForAudioPolicy(mIdentity.getIsolatedUid()); 392 } 393 mIdentity = null; 394 } 395 } 396 397 @SuppressWarnings("GuardedBy") updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, @NonNull IBinder token)398 void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, 399 @NonNull IBinder token) { 400 final DetectorSession session = getDetectorSessionByTokenLocked(token); 401 if (session == null) { 402 Slog.v(TAG, "Not found the detector by token"); 403 return; 404 } 405 session.updateStateLocked(options, sharedMemory, mLastRestartInstant); 406 } 407 408 /** 409 * This method is only used by SoftwareHotwordDetector. 410 */ startListeningFromMicLocked( AudioFormat audioFormat, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)411 void startListeningFromMicLocked( 412 AudioFormat audioFormat, 413 IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { 414 if (DEBUG) { 415 Slog.d(TAG, "startListeningFromMicLocked"); 416 } 417 // We only support one Dsp trusted hotword detector and one software hotword detector at 418 // the same time, so we can reuse original single software trusted hotword mechanism. 419 final SoftwareTrustedHotwordDetectorSession session = 420 getSoftwareTrustedHotwordDetectorSessionLocked(); 421 if (session == null) { 422 return; 423 } 424 session.startListeningFromMicLocked(audioFormat, callback); 425 } 426 setVisualQueryDetectionAttentionListenerLocked( @ullable IVisualQueryDetectionAttentionListener listener)427 public void setVisualQueryDetectionAttentionListenerLocked( 428 @Nullable IVisualQueryDetectionAttentionListener listener) { 429 final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked(); 430 if (session == null) { 431 return; 432 } 433 session.setVisualQueryDetectionAttentionListenerLocked(listener); 434 } 435 436 /** 437 * This method is only used by VisualQueryDetector. 438 */ startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback)439 boolean startPerceivingLocked(IVisualQueryDetectionVoiceInteractionCallback callback) { 440 if (DEBUG) { 441 Slog.d(TAG, "startPerceivingLocked"); 442 } 443 final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked(); 444 if (session == null) { 445 return false; 446 } 447 return session.startPerceivingLocked(callback); 448 } 449 450 /** 451 * This method is only used by VisaulQueryDetector. 452 */ stopPerceivingLocked()453 boolean stopPerceivingLocked() { 454 if (DEBUG) { 455 Slog.d(TAG, "stopPerceivingLocked"); 456 } 457 final VisualQueryDetectorSession session = getVisualQueryDetectorSessionLocked(); 458 if (session == null) { 459 return false; 460 } 461 return session.stopPerceivingLocked(); 462 } 463 startListeningFromExternalSourceLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, @Nullable PersistableBundle options, @NonNull IBinder token, IMicrophoneHotwordDetectionVoiceInteractionCallback callback)464 public void startListeningFromExternalSourceLocked( 465 ParcelFileDescriptor audioStream, 466 AudioFormat audioFormat, 467 @Nullable PersistableBundle options, 468 @NonNull IBinder token, 469 IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { 470 if (DEBUG) { 471 Slog.d(TAG, "startListeningFromExternalSourceLocked"); 472 } 473 final DetectorSession session = getDetectorSessionByTokenLocked(token); 474 if (session == null) { 475 Slog.v(TAG, "Not found the detector by token"); 476 return; 477 } 478 session.startListeningFromExternalSourceLocked(audioStream, audioFormat, options, callback); 479 } 480 startListeningFromWearableLocked( ParcelFileDescriptor audioStream, AudioFormat audioFormat, PersistableBundle options, WearableHotwordDetectionCallback callback)481 public void startListeningFromWearableLocked( 482 ParcelFileDescriptor audioStream, 483 AudioFormat audioFormat, 484 PersistableBundle options, 485 WearableHotwordDetectionCallback callback) { 486 if (DEBUG) { 487 Slog.d(TAG, "startListeningFromWearableLocked"); 488 } 489 DetectorSession trustedSession = getDspTrustedHotwordDetectorSessionLocked(); 490 if (trustedSession == null) { 491 callback.onError( 492 "Unable to start listening from wearable because the trusted hotword detection" 493 + " session is not available."); 494 return; 495 } 496 trustedSession.startListeningFromWearableLocked( 497 audioStream, audioFormat, options, callback); 498 } 499 500 /** 501 * This method is only used by SoftwareHotwordDetector. 502 */ stopListeningFromMicLocked()503 void stopListeningFromMicLocked() { 504 if (DEBUG) { 505 Slog.d(TAG, "stopListeningFromMicLocked"); 506 } 507 final SoftwareTrustedHotwordDetectorSession session = 508 getSoftwareTrustedHotwordDetectorSessionLocked(); 509 if (session == null) { 510 return; 511 } 512 session.stopListeningFromMicLocked(); 513 } 514 triggerHardwareRecognitionEventForTestLocked( SoundTrigger.KeyphraseRecognitionEvent event, IHotwordRecognitionStatusCallback callback)515 void triggerHardwareRecognitionEventForTestLocked( 516 SoundTrigger.KeyphraseRecognitionEvent event, 517 IHotwordRecognitionStatusCallback callback) { 518 if (DEBUG) { 519 Slog.d(TAG, "triggerHardwareRecognitionEventForTestLocked"); 520 } 521 detectFromDspSource(event, callback); 522 } 523 detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, IHotwordRecognitionStatusCallback externalCallback)524 private void detectFromDspSource(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, 525 IHotwordRecognitionStatusCallback externalCallback) { 526 if (DEBUG) { 527 Slog.d(TAG, "detectFromDspSource"); 528 } 529 // We only support one Dsp trusted hotword detector and one software hotword detector at 530 // the same time, so we can reuse original single Dsp trusted hotword mechanism. 531 synchronized (mLock) { 532 final DspTrustedHotwordDetectorSession session = 533 getDspTrustedHotwordDetectorSessionLocked(); 534 if (session == null || !session.isSameCallback(externalCallback)) { 535 Slog.v(TAG, "Not found the Dsp detector by callback"); 536 return; 537 } 538 session.detectFromDspSourceLocked(recognitionEvent, externalCallback); 539 } 540 } 541 forceRestart()542 void forceRestart() { 543 Slog.v(TAG, "Requested to restart the service internally. Performing the restart"); 544 synchronized (mLock) { 545 restartProcessLocked(); 546 } 547 } 548 549 @SuppressWarnings("GuardedBy") setDebugHotwordLoggingLocked(boolean logging)550 void setDebugHotwordLoggingLocked(boolean logging) { 551 Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); 552 clearDebugHotwordLoggingTimeoutLocked(); 553 mDebugHotwordLogging = logging; 554 runForEachDetectorSessionLocked((session) -> { 555 session.setDebugHotwordLoggingLocked(logging); 556 }); 557 558 if (logging) { 559 // Reset mDebugHotwordLogging to false after one hour 560 mDebugHotwordLoggingTimeoutFuture = mScheduledExecutorService.schedule(() -> { 561 Slog.v(TAG, "Timeout to reset mDebugHotwordLogging to false"); 562 synchronized (mLock) { 563 mDebugHotwordLogging = false; 564 runForEachDetectorSessionLocked((session) -> { 565 session.setDebugHotwordLoggingLocked(false); 566 }); 567 } 568 }, RESET_DEBUG_HOTWORD_LOGGING_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); 569 } 570 } 571 setDetectorType(int detectorType)572 void setDetectorType(int detectorType) { 573 mDetectorType = detectorType; 574 } 575 clearDebugHotwordLoggingTimeoutLocked()576 private void clearDebugHotwordLoggingTimeoutLocked() { 577 if (mDebugHotwordLoggingTimeoutFuture != null) { 578 mDebugHotwordLoggingTimeoutFuture.cancel(/* mayInterruptIfRunning= */ true); 579 mDebugHotwordLoggingTimeoutFuture = null; 580 } 581 } 582 583 @SuppressWarnings("GuardedBy") restartProcessLocked()584 private void restartProcessLocked() { 585 // TODO(b/244598068): Check HotwordAudioStreamManager first 586 Slog.v(TAG, "Restarting hotword detection process"); 587 588 ServiceConnection oldHotwordConnection = mRemoteHotwordDetectionService; 589 ServiceConnection oldVisualQueryDetectionConnection = mRemoteVisualQueryDetectionService; 590 HotwordDetectionServiceIdentity previousIdentity = mIdentity; 591 592 mLastRestartInstant = Instant.now(); 593 // Recreate connection to reset the cache. 594 mRestartCount++; 595 596 if (oldHotwordConnection != null) { 597 mRemoteHotwordDetectionService = 598 mHotwordDetectionServiceConnectionFactory.createLocked(); 599 } 600 601 if (oldVisualQueryDetectionConnection != null) { 602 mRemoteVisualQueryDetectionService = 603 mVisualQueryDetectionServiceConnectionFactory.createLocked(); 604 } 605 606 Slog.v(TAG, "Started the new process, dispatching processRestarted to detector"); 607 runForEachDetectorSessionLocked((session) -> { 608 HotwordDetectionConnection.ServiceConnection newRemoteService = 609 (session instanceof VisualQueryDetectorSession) 610 ? mRemoteVisualQueryDetectionService : mRemoteHotwordDetectionService; 611 session.updateRemoteSandboxedDetectionServiceLocked(newRemoteService); 612 session.informRestartProcessLocked(); 613 }); 614 if (DEBUG) { 615 Slog.i(TAG, "processRestarted is dispatched done, unbinding from the old process"); 616 } 617 618 if (oldHotwordConnection != null) { 619 oldHotwordConnection.ignoreConnectionStatusEvents(); 620 oldHotwordConnection.unbind(); 621 } 622 623 if (oldVisualQueryDetectionConnection != null) { 624 oldVisualQueryDetectionConnection.ignoreConnectionStatusEvents(); 625 oldVisualQueryDetectionConnection.unbind(); 626 } 627 628 // TODO(b/266670431): Handles identity resetting for the new process to make sure the 629 // correct identity is provided. 630 631 if (previousIdentity != null) { 632 removeServiceUidForAudioPolicy(previousIdentity.getIsolatedUid()); 633 } 634 } 635 636 /** 637 * Shutdowns down hotword detection service, swallowing exceptions. 638 * 639 * Called when voice activation app-op has been disabled. 640 */ 641 @SuppressWarnings("GuardedBy") safelyShutdownHotwordDetectionOnVoiceActivationDisabledLocked()642 void safelyShutdownHotwordDetectionOnVoiceActivationDisabledLocked() { 643 Slog.v(TAG, "safelyShutdownHotwordDetectionOnVoiceActivationDisabled"); 644 try { 645 clearDebugHotwordLoggingTimeoutLocked(); 646 mRemoteExceptionListener = null; 647 runForEachDetectorSessionLocked((session) -> { 648 if (!(session instanceof VisualQueryDetectorSession)) { 649 // Inform all detector sessions that they got destroyed due to voice activation 650 // op being disabled. 651 session.reportErrorLocked( 652 new HotwordDetectionServiceFailure( 653 HotwordDetectionServiceFailure 654 .ERROR_CODE_SHUTDOWN_HDS_ON_VOICE_ACTIVATION_OP_DISABLED, 655 "Shutdown hotword detection service on voice " 656 + "activation op disabled!")); 657 session.destroyLocked(); 658 } 659 }); 660 661 // Remove hotword detection sessions. 662 mDetectorSessions.delete(HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP); 663 mDetectorSessions.delete(HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE); 664 665 mDebugHotwordLogging = false; 666 unbindHotwordDetectionService(); 667 if (mCancellationTaskFuture != null) { 668 mCancellationTaskFuture.cancel(/* mayInterruptIfRunning= */ true); 669 } 670 if (mAudioFlinger != null) { 671 mAudioFlinger.unlinkToDeath(mAudioServerDeathRecipient, /* flags= */ 0); 672 mAudioFlinger = null; 673 } 674 } catch (Exception e) { 675 Slog.e(TAG, "Swallowing error while shutting down hotword detection." 676 + "Error message: " + e.getMessage()); 677 } 678 } 679 680 681 static final class SoundTriggerCallback extends IRecognitionStatusCallback.Stub { 682 private final HotwordDetectionConnection mHotwordDetectionConnection; 683 private final IHotwordRecognitionStatusCallback mExternalCallback; 684 private final Identity mVoiceInteractorIdentity; 685 private final Context mContext; 686 SoundTriggerCallback(Context context, IHotwordRecognitionStatusCallback callback, HotwordDetectionConnection connection, Identity voiceInteractorIdentity)687 SoundTriggerCallback(Context context, IHotwordRecognitionStatusCallback callback, 688 HotwordDetectionConnection connection, Identity voiceInteractorIdentity) { 689 mContext = context; 690 mHotwordDetectionConnection = connection; 691 mExternalCallback = callback; 692 mVoiceInteractorIdentity = voiceInteractorIdentity; 693 } 694 695 @Override onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent)696 public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) 697 throws RemoteException { 698 if (DEBUG) { 699 Slog.d(TAG, "onKeyphraseDetected recognitionEvent : " + recognitionEvent); 700 } 701 final boolean useHotwordDetectionService = mHotwordDetectionConnection != null; 702 if (useHotwordDetectionService) { 703 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 704 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__TRUSTED_DETECTOR_DSP, 705 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER, 706 mVoiceInteractorIdentity.uid); 707 mHotwordDetectionConnection.detectFromDspSource( 708 recognitionEvent, mExternalCallback); 709 } else { 710 // We have to attribute ops here, since we configure all st clients as trusted to 711 // enable a partial exemption. 712 // TODO (b/292012931) remove once trusted uniformly required. 713 int result = mContext.getSystemService(AppOpsManager.class) 714 .noteOpNoThrow(AppOpsManager.OP_RECORD_AUDIO_HOTWORD, 715 mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, 716 mVoiceInteractorIdentity.attributionTag, 717 "Non-HDS keyphrase recognition to VoiceInteractionService"); 718 719 if (result != AppOpsManager.MODE_ALLOWED) { 720 Slog.w(TAG, "onKeyphraseDetected suppressed, permission check returned: " 721 + result); 722 mExternalCallback.onRecognitionPaused(); 723 } else { 724 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 725 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__DETECTOR_TYPE__NORMAL_DETECTOR, 726 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__KEYPHRASE_TRIGGER, 727 mVoiceInteractorIdentity.uid); 728 mExternalCallback.onKeyphraseDetected(recognitionEvent, null); 729 } 730 } 731 } 732 733 @Override onGenericSoundTriggerDetected( SoundTrigger.GenericRecognitionEvent recognitionEvent)734 public void onGenericSoundTriggerDetected( 735 SoundTrigger.GenericRecognitionEvent recognitionEvent) 736 throws RemoteException { 737 mExternalCallback.onGenericSoundTriggerDetected(recognitionEvent); 738 } 739 740 @Override onPreempted()741 public void onPreempted() throws RemoteException { 742 mExternalCallback.onSoundTriggerFailure(new SoundTriggerFailure( 743 SoundTriggerFailure.ERROR_CODE_UNEXPECTED_PREEMPTION, 744 "Unexpected startRecognition on already started ST session")); 745 } 746 747 @Override onModuleDied()748 public void onModuleDied() throws RemoteException { 749 mExternalCallback.onSoundTriggerFailure(new SoundTriggerFailure( 750 SoundTriggerFailure.ERROR_CODE_MODULE_DIED, 751 "STHAL died")); 752 } 753 754 @Override onResumeFailed(int status)755 public void onResumeFailed(int status) throws RemoteException { 756 mExternalCallback.onSoundTriggerFailure(new SoundTriggerFailure( 757 SoundTriggerFailure.ERROR_CODE_RECOGNITION_RESUME_FAILED, 758 "STService recognition resume failed with: " + status)); 759 } 760 761 @Override onPauseFailed(int status)762 public void onPauseFailed(int status) throws RemoteException { 763 mExternalCallback.onSoundTriggerFailure(new SoundTriggerFailure( 764 SoundTriggerFailure.ERROR_CODE_RECOGNITION_RESUME_FAILED, 765 "STService recognition pause failed with: " + status)); 766 } 767 768 @Override onRecognitionPaused()769 public void onRecognitionPaused() throws RemoteException { 770 mExternalCallback.onRecognitionPaused(); 771 } 772 773 @Override onRecognitionResumed()774 public void onRecognitionResumed() throws RemoteException { 775 mExternalCallback.onRecognitionResumed(); 776 } 777 } 778 dump(String prefix, PrintWriter pw)779 public void dump(String prefix, PrintWriter pw) { 780 synchronized (mLock) { 781 pw.print(prefix); pw.print("mReStartPeriodSeconds="); pw.println(mReStartPeriodSeconds); 782 pw.print(prefix); pw.print("bound for HotwordDetectionService="); 783 pw.println(mRemoteHotwordDetectionService != null 784 && mRemoteHotwordDetectionService.isBound()); 785 pw.print(prefix); pw.print("bound for VisualQueryDetectionService="); 786 pw.println(mRemoteVisualQueryDetectionService != null 787 && mRemoteHotwordDetectionService != null 788 && mRemoteHotwordDetectionService.isBound()); 789 pw.print(prefix); pw.print("mRestartCount="); 790 pw.println(mRestartCount); 791 pw.print(prefix); pw.print("mLastRestartInstant="); pw.println(mLastRestartInstant); 792 pw.print(prefix); pw.println("DetectorSession(s):"); 793 pw.print(prefix); pw.print("Num of DetectorSession(s)="); 794 pw.println(mDetectorSessions.size()); 795 runForEachDetectorSessionLocked((session) -> { 796 session.dumpLocked(prefix, pw); 797 }); 798 } 799 } 800 801 private class ServiceConnectionFactory { 802 private final Intent mIntent; 803 private final int mBindingFlags; 804 private final int mDetectionServiceType; 805 ServiceConnectionFactory(@onNull Intent intent, boolean bindInstantServiceAllowed, int detectionServiceType)806 ServiceConnectionFactory(@NonNull Intent intent, boolean bindInstantServiceAllowed, 807 int detectionServiceType) { 808 mIntent = intent; 809 mDetectionServiceType = detectionServiceType; 810 int flags = bindInstantServiceAllowed ? Context.BIND_ALLOW_INSTANT : 0; 811 if (SYSPROP_VISUAL_QUERY_SERVICE_ENABLED 812 && mVisualQueryDetectionComponentName != null 813 && mHotwordDetectionComponentName != null) { 814 flags |= Context.BIND_SHARED_ISOLATED_PROCESS; 815 } 816 mBindingFlags = flags; 817 } 818 createLocked()819 ServiceConnection createLocked() { 820 ServiceConnection connection = 821 new ServiceConnection(mContext, mIntent, mBindingFlags, mUserId, 822 ISandboxedDetectionService.Stub::asInterface, 823 mRestartCount % MAX_ISOLATED_PROCESS_NUMBER, mDetectionServiceType); 824 connection.connect(); 825 826 updateAudioFlinger(connection, mAudioFlinger); 827 updateContentCaptureManager(connection); 828 updateSpeechService(connection); 829 updateServiceIdentity(connection); 830 updateStorageService(connection); 831 return connection; 832 } 833 } 834 835 class ServiceConnection extends ServiceConnector.Impl<ISandboxedDetectionService> { 836 private final Object mLock = new Object(); 837 838 private final Intent mIntent; 839 private final int mBindingFlags; 840 private final int mInstanceNumber; 841 842 private static final HandlerThread mHandler; 843 844 static { 845 mHandler = new HandlerThread("Sandbox detection connector"); mHandler.start()846 mHandler.start(); 847 } 848 849 private boolean mRespectServiceConnectionStatusChanged = true; 850 private boolean mIsBound = false; 851 private boolean mIsLoggedFirstConnect = false; 852 private final int mDetectionServiceType; 853 ServiceConnection(@onNull Context context, @NonNull Intent serviceIntent, int bindingFlags, int userId, @Nullable Function<IBinder, ISandboxedDetectionService> binderAsInterface, int instanceNumber, int detectionServiceType)854 ServiceConnection(@NonNull Context context, 855 @NonNull Intent serviceIntent, int bindingFlags, int userId, 856 @Nullable Function<IBinder, ISandboxedDetectionService> binderAsInterface, 857 int instanceNumber, int detectionServiceType) { 858 super(context, serviceIntent, bindingFlags, userId, binderAsInterface); 859 this.mIntent = serviceIntent; 860 this.mBindingFlags = bindingFlags; 861 this.mInstanceNumber = instanceNumber; 862 this.mDetectionServiceType = detectionServiceType; 863 } 864 865 @Override // from ServiceConnector.Impl onServiceConnectionStatusChanged(ISandboxedDetectionService service, boolean connected)866 protected void onServiceConnectionStatusChanged(ISandboxedDetectionService service, 867 boolean connected) { 868 if (DEBUG) { 869 Slog.d(TAG, "onServiceConnectionStatusChanged connected = " + connected); 870 } 871 synchronized (mLock) { 872 if (!mRespectServiceConnectionStatusChanged) { 873 Slog.v(TAG, "Ignored onServiceConnectionStatusChanged event"); 874 return; 875 } 876 mIsBound = connected; 877 878 if (!connected) { 879 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 880 HotwordMetricsLogger.writeDetectorEvent(mDetectorType, 881 HOTWORD_DETECTOR_EVENTS__EVENT__ON_DISCONNECTED, 882 mVoiceInteractionServiceUid); 883 } 884 } else if (!mIsLoggedFirstConnect) { 885 mIsLoggedFirstConnect = true; 886 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 887 HotwordMetricsLogger.writeDetectorEvent(mDetectorType, 888 HOTWORD_DETECTOR_EVENTS__EVENT__ON_CONNECTED, 889 mVoiceInteractionServiceUid); 890 } 891 } 892 } 893 } 894 895 @Override // from ServiceConnector.Impl getJobHandler()896 protected Handler getJobHandler() { 897 return mHandler.getThreadHandler(); 898 } 899 900 @Override getAutoDisconnectTimeoutMs()901 protected long getAutoDisconnectTimeoutMs() { 902 return -1; 903 } 904 905 @Override binderDied()906 public void binderDied() { 907 super.binderDied(); 908 Slog.w(TAG, "binderDied mDetectionServiceType = " + mDetectionServiceType); 909 synchronized (mLock) { 910 if (!mRespectServiceConnectionStatusChanged) { 911 Slog.v(TAG, "Ignored #binderDied event"); 912 return; 913 } 914 } 915 //TODO(b265535257): report error to either service only. 916 synchronized (HotwordDetectionConnection.this.mLock) { 917 runForEachDetectorSessionLocked(this::reportBinderDiedLocked); 918 } 919 // Can improve to log exit reason if needed 920 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 921 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 922 mDetectorType, 923 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__SERVICE_CRASH, 924 mVoiceInteractionServiceUid); 925 } 926 } 927 928 @Override bindService( @onNull android.content.ServiceConnection serviceConnection)929 protected boolean bindService( 930 @NonNull android.content.ServiceConnection serviceConnection) { 931 try { 932 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 933 HotwordMetricsLogger.writeDetectorEvent(mDetectorType, 934 HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE, 935 mVoiceInteractionServiceUid); 936 } 937 boolean bindResult = mContext.bindIsolatedService( 938 mIntent, 939 Context.BIND_AUTO_CREATE | Context.BIND_FOREGROUND_SERVICE | mBindingFlags, 940 "hotword_detector_" + mInstanceNumber, 941 mExecutor, 942 serviceConnection); 943 if (!bindResult) { 944 Slog.w(TAG, 945 "bindService failure mDetectionServiceType = " + mDetectionServiceType); 946 synchronized (HotwordDetectionConnection.this.mLock) { 947 runForEachDetectorSessionLocked(this::reportBindServiceFailureLocked); 948 } 949 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 950 HotwordMetricsLogger.writeDetectorEvent(mDetectorType, 951 HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, 952 mVoiceInteractionServiceUid); 953 } 954 } 955 return bindResult; 956 } catch (IllegalArgumentException e) { 957 if (mDetectionServiceType != DETECTION_SERVICE_TYPE_VISUAL_QUERY) { 958 HotwordMetricsLogger.writeDetectorEvent(mDetectorType, 959 HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_BIND_SERVICE_FAIL, 960 mVoiceInteractionServiceUid); 961 } 962 Slog.wtf(TAG, "Can't bind to the hotword detection service!", e); 963 return false; 964 } 965 } 966 isBound()967 boolean isBound() { 968 synchronized (mLock) { 969 return mIsBound; 970 } 971 } 972 ignoreConnectionStatusEvents()973 void ignoreConnectionStatusEvents() { 974 synchronized (mLock) { 975 mRespectServiceConnectionStatusChanged = false; 976 } 977 } 978 reportBinderDiedLocked(DetectorSession detectorSession)979 private void reportBinderDiedLocked(DetectorSession detectorSession) { 980 if (mDetectionServiceType == DETECTION_SERVICE_TYPE_HOTWORD && ( 981 detectorSession instanceof DspTrustedHotwordDetectorSession 982 || detectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { 983 detectorSession.reportErrorLocked(new HotwordDetectionServiceFailure( 984 HotwordDetectionServiceFailure.ERROR_CODE_BINDING_DIED, 985 "Detection service is dead.")); 986 } else if (mDetectionServiceType == DETECTION_SERVICE_TYPE_VISUAL_QUERY 987 && detectorSession instanceof VisualQueryDetectorSession) { 988 detectorSession.reportErrorLocked(new VisualQueryDetectionServiceFailure( 989 VisualQueryDetectionServiceFailure.ERROR_CODE_BINDING_DIED, 990 "Detection service is dead.")); 991 } else { 992 detectorSession.reportErrorLocked( 993 "Detection service is dead with unknown detection service type."); 994 } 995 } 996 reportBindServiceFailureLocked(DetectorSession detectorSession)997 private void reportBindServiceFailureLocked(DetectorSession detectorSession) { 998 if (mDetectionServiceType == DETECTION_SERVICE_TYPE_HOTWORD && ( 999 detectorSession instanceof DspTrustedHotwordDetectorSession 1000 || detectorSession instanceof SoftwareTrustedHotwordDetectorSession)) { 1001 detectorSession.reportErrorLocked(new HotwordDetectionServiceFailure( 1002 HotwordDetectionServiceFailure.ERROR_CODE_BIND_FAILURE, 1003 "Bind detection service failure.")); 1004 } else if (mDetectionServiceType == DETECTION_SERVICE_TYPE_VISUAL_QUERY 1005 && detectorSession instanceof VisualQueryDetectorSession) { 1006 detectorSession.reportErrorLocked(new VisualQueryDetectionServiceFailure( 1007 VisualQueryDetectionServiceFailure.ERROR_CODE_BIND_FAILURE, 1008 "Bind detection service failure.")); 1009 } else { 1010 detectorSession.reportErrorLocked( 1011 "Bind detection service failure with unknown detection service type."); 1012 } 1013 } 1014 } 1015 1016 @SuppressWarnings("GuardedBy") createDetectorLocked( @ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @NonNull IBinder token, @NonNull IHotwordRecognitionStatusCallback callback, int detectorType)1017 void createDetectorLocked( 1018 @Nullable PersistableBundle options, 1019 @Nullable SharedMemory sharedMemory, 1020 @NonNull IBinder token, 1021 @NonNull IHotwordRecognitionStatusCallback callback, 1022 int detectorType) { 1023 // We only support one Dsp trusted hotword detector and one software hotword detector at 1024 // the same time, remove existing one. 1025 DetectorSession removeSession = mDetectorSessions.get(detectorType); 1026 if (removeSession != null) { 1027 removeSession.destroyLocked(); 1028 mDetectorSessions.remove(detectorType); 1029 } 1030 final DetectorSession session; 1031 if (detectorType == HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP) { 1032 if (mRemoteHotwordDetectionService == null) { 1033 mRemoteHotwordDetectionService = 1034 mHotwordDetectionServiceConnectionFactory.createLocked(); 1035 } 1036 session = new DspTrustedHotwordDetectorSession(mRemoteHotwordDetectionService, 1037 mLock, mContext, token, callback, mVoiceInteractionServiceUid, 1038 mVoiceInteractorIdentity, mScheduledExecutorService, mDebugHotwordLogging, 1039 mRemoteExceptionListener, mUserId); 1040 } else if (detectorType == HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { 1041 if (mRemoteVisualQueryDetectionService == null) { 1042 mRemoteVisualQueryDetectionService = 1043 mVisualQueryDetectionServiceConnectionFactory.createLocked(); 1044 } 1045 session = new VisualQueryDetectorSession( 1046 mRemoteVisualQueryDetectionService, mLock, mContext, token, callback, 1047 mVoiceInteractionServiceUid, mVoiceInteractorIdentity, 1048 mScheduledExecutorService, mDebugHotwordLogging, mRemoteExceptionListener, 1049 mUserId); 1050 } else { 1051 if (mRemoteHotwordDetectionService == null) { 1052 mRemoteHotwordDetectionService = 1053 mHotwordDetectionServiceConnectionFactory.createLocked(); 1054 } 1055 session = new SoftwareTrustedHotwordDetectorSession( 1056 mRemoteHotwordDetectionService, mLock, mContext, token, callback, 1057 mVoiceInteractionServiceUid, mVoiceInteractorIdentity, 1058 mScheduledExecutorService, mDebugHotwordLogging, 1059 mRemoteExceptionListener, mUserId); 1060 } 1061 mHotwordRecognitionCallback = callback; 1062 mDetectorSessions.put(detectorType, session); 1063 session.initialize(options, sharedMemory); 1064 } 1065 1066 @SuppressWarnings("GuardedBy") destroyDetectorLocked(@onNull IBinder token)1067 void destroyDetectorLocked(@NonNull IBinder token) { 1068 final DetectorSession session = getDetectorSessionByTokenLocked(token); 1069 if (session == null) { 1070 return; 1071 } 1072 session.destroyLocked(); 1073 final int index = mDetectorSessions.indexOfValue(session); 1074 if (index < 0 || index > mDetectorSessions.size() - 1) { 1075 return; 1076 } 1077 mDetectorSessions.removeAt(index); 1078 if (session instanceof VisualQueryDetectorSession) { 1079 unbindVisualQueryDetectionService(); 1080 } 1081 // Handle case where all hotword detector sessions are destroyed with only the visual 1082 // detector session left 1083 boolean allHotwordDetectionServiceSessionsRemoved = mDetectorSessions.size() == 0 1084 || (mDetectorSessions.size() == 1 && mDetectorSessions.get(0) 1085 instanceof VisualQueryDetectorSession); 1086 if (allHotwordDetectionServiceSessionsRemoved) { 1087 unbindHotwordDetectionService(); 1088 } 1089 } 1090 1091 @SuppressWarnings("GuardedBy") getDetectorSessionByTokenLocked(IBinder token)1092 private DetectorSession getDetectorSessionByTokenLocked(IBinder token) { 1093 if (token == null) { 1094 return null; 1095 } 1096 for (int i = 0; i < mDetectorSessions.size(); i++) { 1097 final DetectorSession session = mDetectorSessions.valueAt(i); 1098 if (!session.isDestroyed() && session.isSameToken(token)) { 1099 return session; 1100 } 1101 } 1102 return null; 1103 } 1104 1105 @SuppressWarnings("GuardedBy") getDspTrustedHotwordDetectorSessionLocked()1106 private DspTrustedHotwordDetectorSession getDspTrustedHotwordDetectorSessionLocked() { 1107 final DetectorSession session = mDetectorSessions.get( 1108 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP); 1109 if (session == null || session.isDestroyed()) { 1110 Slog.v(TAG, "Not found the Dsp detector"); 1111 return null; 1112 } 1113 return (DspTrustedHotwordDetectorSession) session; 1114 } 1115 1116 @SuppressWarnings("GuardedBy") getSoftwareTrustedHotwordDetectorSessionLocked()1117 private SoftwareTrustedHotwordDetectorSession getSoftwareTrustedHotwordDetectorSessionLocked() { 1118 final DetectorSession session = mDetectorSessions.get( 1119 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE); 1120 if (session == null || session.isDestroyed()) { 1121 Slog.v(TAG, "Not found the software detector"); 1122 return null; 1123 } 1124 return (SoftwareTrustedHotwordDetectorSession) session; 1125 } 1126 1127 @SuppressWarnings("GuardedBy") getVisualQueryDetectorSessionLocked()1128 private VisualQueryDetectorSession getVisualQueryDetectorSessionLocked() { 1129 final DetectorSession session = mDetectorSessions.get( 1130 HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR); 1131 if (session == null || session.isDestroyed()) { 1132 Slog.v(TAG, "Not found the visual query detector"); 1133 return null; 1134 } 1135 return (VisualQueryDetectorSession) session; 1136 } runForEachDetectorSessionLocked( @onNull Consumer<DetectorSession> action)1137 private void runForEachDetectorSessionLocked( 1138 @NonNull Consumer<DetectorSession> action) { 1139 for (int i = 0; i < mDetectorSessions.size(); i++) { 1140 DetectorSession session = mDetectorSessions.valueAt(i); 1141 action.accept(session); 1142 } 1143 } 1144 updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger)1145 private static void updateAudioFlinger(ServiceConnection connection, IBinder audioFlinger) { 1146 // TODO: Consider using a proxy that limits the exposed API surface. 1147 connection.run(service -> service.updateAudioFlinger(audioFlinger)); 1148 } 1149 updateContentCaptureManager(ServiceConnection connection)1150 private static void updateContentCaptureManager(ServiceConnection connection) { 1151 IBinder b = ServiceManager 1152 .getService(Context.CONTENT_CAPTURE_MANAGER_SERVICE); 1153 IContentCaptureManager binderService = IContentCaptureManager.Stub.asInterface(b); 1154 connection.run( 1155 service -> service.updateContentCaptureManager(binderService, 1156 new ContentCaptureOptions(null))); 1157 } 1158 updateSpeechService(ServiceConnection connection)1159 private static void updateSpeechService(ServiceConnection connection) { 1160 IBinder b = ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE); 1161 IRecognitionServiceManager binderService = IRecognitionServiceManager.Stub.asInterface(b); 1162 connection.run(service -> { 1163 service.updateRecognitionServiceManager(binderService); 1164 }); 1165 } 1166 updateServiceIdentity(ServiceConnection connection)1167 private void updateServiceIdentity(ServiceConnection connection) { 1168 connection.run(service -> service.ping(new ISandboxedDetectionService.IPingMe.Stub() { 1169 @Override 1170 public void onPing() throws RemoteException { 1171 // TODO: Exit if the service has been unbound already (though there's a very low 1172 // chance this happens). 1173 Slog.d(TAG, "updating hotword UID " + Binder.getCallingUid()); 1174 // TODO: Have the provider point to the current state stored in 1175 // VoiceInteractionManagerServiceImpl. 1176 final int uid = Binder.getCallingUid(); 1177 LocalServices.getService(PermissionManagerServiceInternal.class) 1178 .setHotwordDetectionServiceProvider(() -> uid); 1179 mIdentity = new HotwordDetectionServiceIdentity(uid, mVoiceInteractionServiceUid); 1180 addServiceUidForAudioPolicy(uid, mVoiceInteractionServiceUid); 1181 } 1182 })); 1183 } 1184 updateStorageService(ServiceConnection connection)1185 private void updateStorageService(ServiceConnection connection) { 1186 connection.run(service -> { 1187 service.registerRemoteStorageService(new IDetectorSessionStorageService.Stub() { 1188 @Override 1189 public void openFile(String filename, AndroidFuture future) 1190 throws RemoteException { 1191 Slog.v(TAG, "BinderCallback#onFileOpen"); 1192 try { 1193 mHotwordRecognitionCallback.onOpenFile(filename, future); 1194 } catch (RemoteException e) { 1195 e.rethrowFromSystemServer(); 1196 } 1197 } 1198 }); 1199 }); 1200 } 1201 addServiceUidForAudioPolicy(int isolatedUid, int owningUid)1202 private void addServiceUidForAudioPolicy(int isolatedUid, int owningUid) { 1203 AudioManagerInternal audioManager = LocalServices.getService(AudioManagerInternal.class); 1204 if (audioManager != null) { 1205 audioManager.addAssistantServiceUid(isolatedUid, owningUid); 1206 } 1207 } 1208 removeServiceUidForAudioPolicy(int uid)1209 private void removeServiceUidForAudioPolicy(int uid) { 1210 AudioManagerInternal audioManager = LocalServices.getService(AudioManagerInternal.class); 1211 if (audioManager != null) { 1212 audioManager.removeAssistantServiceUid(uid); 1213 } 1214 } 1215 } 1216