1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.voiceinteraction; 18 19 import static android.service.voice.HotwordDetectionServiceFailure.ERROR_CODE_DETECT_TIMEOUT; 20 import static android.service.voice.HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_SECURITY_EXCEPTION; 21 import static android.service.voice.HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_STREAM_COPY_FAILURE; 22 23 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_DETECTED_EXCEPTION; 24 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; 25 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION; 26 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION; 27 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED; 28 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT; 29 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED; 30 import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART; 31 32 import android.annotation.NonNull; 33 import android.content.Context; 34 import android.hardware.soundtrigger.SoundTrigger; 35 import android.media.permission.Identity; 36 import android.os.IBinder; 37 import android.os.PersistableBundle; 38 import android.os.RemoteException; 39 import android.os.SharedMemory; 40 import android.service.voice.HotwordDetectedResult; 41 import android.service.voice.HotwordDetectionService; 42 import android.service.voice.HotwordDetectionServiceFailure; 43 import android.service.voice.HotwordDetector; 44 import android.service.voice.HotwordRejectedResult; 45 import android.service.voice.IDspHotwordDetectionCallback; 46 import android.util.Slog; 47 48 import com.android.internal.annotations.GuardedBy; 49 import com.android.internal.app.IHotwordRecognitionStatusCallback; 50 import com.android.server.voiceinteraction.VoiceInteractionManagerServiceImpl.DetectorRemoteExceptionListener; 51 52 import java.io.IOException; 53 import java.io.PrintWriter; 54 import java.util.Locale; 55 import java.util.concurrent.ScheduledExecutorService; 56 import java.util.concurrent.ScheduledFuture; 57 import java.util.concurrent.TimeUnit; 58 import java.util.concurrent.atomic.AtomicBoolean; 59 60 /** 61 * A class that provides Dsp trusted hotword detector to communicate with the {@link 62 * HotwordDetectionService}. 63 * 64 * This class can handle the hotword detection which detector is created by using 65 * {@link android.service.voice.VoiceInteractionService#createAlwaysOnHotwordDetector(String, 66 * Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)}. 67 */ 68 final class DspTrustedHotwordDetectorSession extends DetectorSession { 69 private static final String TAG = "DspTrustedHotwordDetectorSession"; 70 71 // The validation timeout value is 3 seconds for onDetect of DSP trigger event. 72 private static final long VALIDATION_TIMEOUT_MILLIS = 3000; 73 // Write the onDetect timeout metric when it takes more time than MAX_VALIDATION_TIMEOUT_MILLIS. 74 private static final long MAX_VALIDATION_TIMEOUT_MILLIS = 4000; 75 76 @GuardedBy("mLock") 77 private ScheduledFuture<?> mCancellationKeyPhraseDetectionFuture; 78 79 @GuardedBy("mLock") 80 private boolean mValidatingDspTrigger = false; 81 @GuardedBy("mLock") 82 private HotwordRejectedResult mLastHotwordRejectedResult = null; 83 DspTrustedHotwordDetectorSession( @onNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, Identity voiceInteractorIdentity, @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, @NonNull DetectorRemoteExceptionListener listener, int userId)84 DspTrustedHotwordDetectorSession( 85 @NonNull HotwordDetectionConnection.ServiceConnection remoteHotwordDetectionService, 86 @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, 87 @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, 88 Identity voiceInteractorIdentity, 89 @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging, 90 @NonNull DetectorRemoteExceptionListener listener, int userId) { 91 super(remoteHotwordDetectionService, lock, context, token, callback, 92 voiceInteractionServiceUid, voiceInteractorIdentity, scheduledExecutorService, 93 logging, listener, userId); 94 } 95 96 @SuppressWarnings("GuardedBy") detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, IHotwordRecognitionStatusCallback externalCallback)97 void detectFromDspSourceLocked(SoundTrigger.KeyphraseRecognitionEvent recognitionEvent, 98 IHotwordRecognitionStatusCallback externalCallback) { 99 if (DEBUG) { 100 Slog.d(TAG, "detectFromDspSourceLocked"); 101 } 102 103 AtomicBoolean timeoutDetected = new AtomicBoolean(false); 104 // TODO: consider making this a non-anonymous class. 105 IDspHotwordDetectionCallback internalCallback = new IDspHotwordDetectionCallback.Stub() { 106 @Override 107 public void onDetected(HotwordDetectedResult result) throws RemoteException { 108 if (DEBUG) { 109 Slog.d(TAG, "onDetected"); 110 } 111 synchronized (mLock) { 112 if (mCancellationKeyPhraseDetectionFuture != null) { 113 mCancellationKeyPhraseDetectionFuture.cancel(true); 114 } 115 if (timeoutDetected.get()) { 116 return; 117 } 118 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 119 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 120 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECTED, 121 mVoiceInteractionServiceUid); 122 if (!mValidatingDspTrigger) { 123 Slog.i(TAG, "Ignoring #onDetected due to a process restart or previous" 124 + " #onRejected result = " + mLastHotwordRejectedResult); 125 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 126 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 127 METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK, 128 mVoiceInteractionServiceUid); 129 return; 130 } 131 mValidatingDspTrigger = false; 132 try { 133 enforcePermissionsForDataDelivery(); 134 enforceExtraKeyphraseIdNotLeaked(result, recognitionEvent); 135 } catch (SecurityException e) { 136 Slog.w(TAG, "Ignoring #onDetected due to a SecurityException", e); 137 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 138 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 139 METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION, 140 mVoiceInteractionServiceUid); 141 try { 142 externalCallback.onHotwordDetectionServiceFailure( 143 new HotwordDetectionServiceFailure( 144 ERROR_CODE_ON_DETECTED_SECURITY_EXCEPTION, 145 "Security exception occurs in #onDetected method.")); 146 } catch (RemoteException e1) { 147 notifyOnDetectorRemoteException(); 148 HotwordMetricsLogger.writeDetectorEvent( 149 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 150 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, 151 mVoiceInteractionServiceUid); 152 throw e1; 153 } 154 return; 155 } 156 saveProximityValueToBundle(result); 157 HotwordDetectedResult newResult; 158 try { 159 newResult = mHotwordAudioStreamCopier.startCopyingAudioStreams(result); 160 } catch (IOException e) { 161 try { 162 Slog.w(TAG, "Ignoring #onDetected due to a IOException", e); 163 externalCallback.onHotwordDetectionServiceFailure( 164 new HotwordDetectionServiceFailure( 165 ERROR_CODE_ON_DETECTED_STREAM_COPY_FAILURE, 166 "Copy audio stream failure.")); 167 } catch (RemoteException e1) { 168 notifyOnDetectorRemoteException(); 169 throw e1; 170 } 171 return; 172 } 173 try { 174 externalCallback.onKeyphraseDetected(recognitionEvent, newResult); 175 Slog.i(TAG, "Egressed " + HotwordDetectedResult.getUsageSize(newResult) 176 + " bits from hotword trusted process"); 177 } catch (RemoteException e) { 178 notifyOnDetectorRemoteException(); 179 HotwordMetricsLogger.writeDetectorEvent( 180 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 181 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_DETECTED_EXCEPTION, 182 mVoiceInteractionServiceUid); 183 throw e; 184 } 185 if (mDebugHotwordLogging) { 186 Slog.i(TAG, "Egressed detected result: " + newResult); 187 } 188 } 189 } 190 191 @Override 192 public void onRejected(HotwordRejectedResult result) throws RemoteException { 193 if (DEBUG) { 194 Slog.d(TAG, "onRejected"); 195 } 196 synchronized (mLock) { 197 if (mCancellationKeyPhraseDetectionFuture != null) { 198 mCancellationKeyPhraseDetectionFuture.cancel(true); 199 } 200 if (timeoutDetected.get()) { 201 return; 202 } 203 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 204 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 205 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED, 206 mVoiceInteractionServiceUid); 207 if (!mValidatingDspTrigger) { 208 Slog.i(TAG, "Ignoring #onRejected due to a process restart"); 209 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 210 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 211 METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK, 212 mVoiceInteractionServiceUid); 213 return; 214 } 215 mValidatingDspTrigger = false; 216 try { 217 externalCallback.onRejected(result); 218 } catch (RemoteException e) { 219 notifyOnDetectorRemoteException(); 220 HotwordMetricsLogger.writeDetectorEvent( 221 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 222 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, 223 mVoiceInteractionServiceUid); 224 throw e; 225 } 226 mLastHotwordRejectedResult = result; 227 if (mDebugHotwordLogging && result != null) { 228 Slog.i(TAG, "Egressed rejected result: " + result); 229 } 230 } 231 } 232 }; 233 234 mValidatingDspTrigger = true; 235 mLastHotwordRejectedResult = null; 236 mRemoteDetectionService.run(service -> { 237 // We use the VALIDATION_TIMEOUT_MILLIS to inform that the client needs to invoke 238 // the callback before timeout value. In order to reduce the latency impact between 239 // server side and client side, we need to use another timeout value 240 // MAX_VALIDATION_TIMEOUT_MILLIS to monitor it. 241 mCancellationKeyPhraseDetectionFuture = mScheduledExecutorService.schedule( 242 () -> { 243 // TODO: avoid allocate every time 244 timeoutDetected.set(true); 245 Slog.w(TAG, "Timed out on #detectFromDspSource"); 246 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 247 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 248 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_TIMEOUT, 249 mVoiceInteractionServiceUid); 250 try { 251 externalCallback.onHotwordDetectionServiceFailure( 252 new HotwordDetectionServiceFailure(ERROR_CODE_DETECT_TIMEOUT, 253 "Timeout to response to the detection result.")); 254 } catch (RemoteException e) { 255 Slog.w(TAG, "Failed to report onError status: ", e); 256 HotwordMetricsLogger.writeDetectorEvent( 257 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 258 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, 259 mVoiceInteractionServiceUid); 260 notifyOnDetectorRemoteException(); 261 } 262 }, 263 MAX_VALIDATION_TIMEOUT_MILLIS, 264 TimeUnit.MILLISECONDS); 265 service.detectFromDspSource( 266 recognitionEvent, 267 recognitionEvent.getCaptureFormat(), 268 VALIDATION_TIMEOUT_MILLIS, 269 internalCallback); 270 }); 271 } 272 273 @Override 274 @SuppressWarnings("GuardedBy") informRestartProcessLocked()275 void informRestartProcessLocked() { 276 // TODO(b/244598068): Check HotwordAudioStreamManager first 277 Slog.v(TAG, "informRestartProcessLocked"); 278 if (mValidatingDspTrigger) { 279 // We're restarting the process while it's processing a DSP trigger, so report a 280 // rejection. This also allows the Interactor to startRecognition again 281 try { 282 mCallback.onRejected(new HotwordRejectedResult.Builder().build()); 283 HotwordMetricsLogger.writeKeyphraseTriggerEvent( 284 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 285 HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECTED_FROM_RESTART, 286 mVoiceInteractionServiceUid); 287 } catch (RemoteException e) { 288 Slog.w(TAG, "Failed to call #rejected"); 289 HotwordMetricsLogger.writeDetectorEvent( 290 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 291 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_REJECTED_EXCEPTION, 292 mVoiceInteractionServiceUid); 293 notifyOnDetectorRemoteException(); 294 } 295 mValidatingDspTrigger = false; 296 } 297 mUpdateStateAfterStartFinished.set(false); 298 299 try { 300 mCallback.onProcessRestarted(); 301 } catch (RemoteException e) { 302 Slog.w(TAG, "Failed to communicate #onProcessRestarted", e); 303 HotwordMetricsLogger.writeDetectorEvent( 304 HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, 305 HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_PROCESS_RESTARTED_EXCEPTION, 306 mVoiceInteractionServiceUid); 307 notifyOnDetectorRemoteException(); 308 } 309 310 mPerformingExternalSourceHotwordDetection = false; 311 closeExternalAudioStreamLocked("process restarted"); 312 } 313 314 @SuppressWarnings("GuardedBy") dumpLocked(String prefix, PrintWriter pw)315 public void dumpLocked(String prefix, PrintWriter pw) { 316 super.dumpLocked(prefix, pw); 317 pw.print(prefix); pw.print("mValidatingDspTrigger="); pw.println(mValidatingDspTrigger); 318 } 319 } 320