1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.app.Activity; 23 import android.content.Intent; 24 import android.hardware.soundtrigger.IRecognitionStatusCallback; 25 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 26 import android.hardware.soundtrigger.KeyphraseMetadata; 27 import android.hardware.soundtrigger.SoundTrigger; 28 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 29 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 30 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 31 import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 32 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 33 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 34 import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; 35 import android.media.AudioFormat; 36 import android.os.AsyncTask; 37 import android.os.Handler; 38 import android.os.Message; 39 import android.os.RemoteException; 40 import android.util.Slog; 41 42 import com.android.internal.app.IVoiceInteractionManagerService; 43 44 import java.io.PrintWriter; 45 import java.lang.annotation.Retention; 46 import java.lang.annotation.RetentionPolicy; 47 import java.util.Locale; 48 49 /** 50 * A class that lets a VoiceInteractionService implementation interact with 51 * always-on keyphrase detection APIs. 52 */ 53 public class AlwaysOnHotwordDetector { 54 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 55 /** 56 * Indicates that this hotword detector is no longer valid for any recognition 57 * and should not be used anymore. 58 */ 59 private static final int STATE_INVALID = -3; 60 61 /** 62 * Indicates that recognition for the given keyphrase is not available on the system 63 * because of the hardware configuration. 64 * No further interaction should be performed with the detector that returns this availability. 65 */ 66 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 67 /** 68 * Indicates that recognition for the given keyphrase is not supported. 69 * No further interaction should be performed with the detector that returns this availability. 70 */ 71 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 72 /** 73 * Indicates that the given keyphrase is not enrolled. 74 * The caller may choose to begin an enrollment flow for the keyphrase. 75 */ 76 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 77 /** 78 * Indicates that the given keyphrase is currently enrolled and it's possible to start 79 * recognition for it. 80 */ 81 public static final int STATE_KEYPHRASE_ENROLLED = 2; 82 83 /** 84 * Indicates that the detector isn't ready currently. 85 */ 86 private static final int STATE_NOT_READY = 0; 87 88 // Keyphrase management actions. Used in getManageIntent() ----// 89 @Retention(RetentionPolicy.SOURCE) 90 @IntDef(prefix = { "MANAGE_ACTION_" }, value = { 91 MANAGE_ACTION_ENROLL, 92 MANAGE_ACTION_RE_ENROLL, 93 MANAGE_ACTION_UN_ENROLL 94 }) 95 private @interface ManageActions {} 96 97 /** 98 * Indicates that we need to enroll. 99 * 100 * @hide 101 */ 102 public static final int MANAGE_ACTION_ENROLL = 0; 103 /** 104 * Indicates that we need to re-enroll. 105 * 106 * @hide 107 */ 108 public static final int MANAGE_ACTION_RE_ENROLL = 1; 109 /** 110 * Indicates that we need to un-enroll. 111 * 112 * @hide 113 */ 114 public static final int MANAGE_ACTION_UN_ENROLL = 2; 115 116 //-- Flags for startRecognition ----// 117 /** @hide */ 118 @Retention(RetentionPolicy.SOURCE) 119 @IntDef(flag = true, prefix = { "RECOGNITION_FLAG_" }, value = { 120 RECOGNITION_FLAG_NONE, 121 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 122 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 123 }) 124 public @interface RecognitionFlags {} 125 126 /** 127 * Empty flag for {@link #startRecognition(int)}. 128 * 129 * @hide 130 */ 131 public static final int RECOGNITION_FLAG_NONE = 0; 132 /** 133 * Recognition flag for {@link #startRecognition(int)} that indicates 134 * whether the trigger audio for hotword needs to be captured. 135 */ 136 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 137 /** 138 * Recognition flag for {@link #startRecognition(int)} that indicates 139 * whether the recognition should keep going on even after the keyphrase triggers. 140 * If this flag is specified, it's possible to get multiple triggers after a 141 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 142 * When this isn't specified, the default behavior is to stop recognition once the 143 * keyphrase is spoken, till the caller starts recognition again. 144 */ 145 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 146 147 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 148 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 149 150 /** @hide */ 151 @Retention(RetentionPolicy.SOURCE) 152 @IntDef(flag = true, prefix = { "RECOGNITION_MODE_" }, value = { 153 RECOGNITION_MODE_VOICE_TRIGGER, 154 RECOGNITION_MODE_USER_IDENTIFICATION, 155 }) 156 public @interface RecognitionModes {} 157 158 /** 159 * Simple recognition of the key phrase. 160 * Returned by {@link #getSupportedRecognitionModes()} 161 */ 162 public static final int RECOGNITION_MODE_VOICE_TRIGGER 163 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 164 /** 165 * User identification performed with the keyphrase recognition. 166 * Returned by {@link #getSupportedRecognitionModes()} 167 */ 168 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 169 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 170 171 static final String TAG = "AlwaysOnHotwordDetector"; 172 static final boolean DBG = false; 173 174 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 175 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 176 177 private static final int MSG_AVAILABILITY_CHANGED = 1; 178 private static final int MSG_HOTWORD_DETECTED = 2; 179 private static final int MSG_DETECTION_ERROR = 3; 180 private static final int MSG_DETECTION_PAUSE = 4; 181 private static final int MSG_DETECTION_RESUME = 5; 182 183 private final String mText; 184 private final Locale mLocale; 185 /** 186 * The metadata of the Keyphrase, derived from the enrollment application. 187 * This may be null if this keyphrase isn't supported by the enrollment application. 188 */ 189 private final KeyphraseMetadata mKeyphraseMetadata; 190 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 191 private final IVoiceInteractionService mVoiceInteractionService; 192 private final IVoiceInteractionManagerService mModelManagementService; 193 private final SoundTriggerListener mInternalCallback; 194 private final Callback mExternalCallback; 195 private final Object mLock = new Object(); 196 private final Handler mHandler; 197 198 private int mAvailability = STATE_NOT_READY; 199 200 /** 201 * Additional payload for {@link Callback#onDetected}. 202 */ 203 public static class EventPayload { 204 private final boolean mTriggerAvailable; 205 // Indicates if {@code captureSession} can be used to continue capturing more audio 206 // from the DSP hardware. 207 private final boolean mCaptureAvailable; 208 // The session to use when attempting to capture more audio from the DSP hardware. 209 private final int mCaptureSession; 210 private final AudioFormat mAudioFormat; 211 // Raw data associated with the event. 212 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 213 private final byte[] mData; 214 EventPayload(boolean triggerAvailable, boolean captureAvailable, AudioFormat audioFormat, int captureSession, byte[] data)215 private EventPayload(boolean triggerAvailable, boolean captureAvailable, 216 AudioFormat audioFormat, int captureSession, byte[] data) { 217 mTriggerAvailable = triggerAvailable; 218 mCaptureAvailable = captureAvailable; 219 mCaptureSession = captureSession; 220 mAudioFormat = audioFormat; 221 mData = data; 222 } 223 224 /** 225 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 226 * May be null if there's no audio present. 227 */ 228 @Nullable getCaptureAudioFormat()229 public AudioFormat getCaptureAudioFormat() { 230 return mAudioFormat; 231 } 232 233 /** 234 * Gets the raw audio that triggered the keyphrase. 235 * This may be null if the trigger audio isn't available. 236 * If non-null, the format of the audio can be obtained by calling 237 * {@link #getCaptureAudioFormat()}. 238 * 239 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 240 */ 241 @Nullable getTriggerAudio()242 public byte[] getTriggerAudio() { 243 if (mTriggerAvailable) { 244 return mData; 245 } else { 246 return null; 247 } 248 } 249 250 /** 251 * Gets the session ID to start a capture from the DSP. 252 * This may be null if streaming capture isn't possible. 253 * If non-null, the format of the audio that can be captured can be 254 * obtained using {@link #getCaptureAudioFormat()}. 255 * 256 * TODO: Candidate for Public API when the API to start capture with a session ID 257 * is made public. 258 * 259 * TODO: Add this to {@link #getCaptureAudioFormat()}: 260 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 261 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 262 * for either the trigger or a streaming session." 263 * 264 * TODO: Should this return a known invalid value instead? 265 * 266 * @hide 267 */ 268 @Nullable getCaptureSession()269 public Integer getCaptureSession() { 270 if (mCaptureAvailable) { 271 return mCaptureSession; 272 } else { 273 return null; 274 } 275 } 276 } 277 278 /** 279 * Callbacks for always-on hotword detection. 280 */ 281 public static abstract class Callback { 282 /** 283 * Called when the hotword availability changes. 284 * This indicates a change in the availability of recognition for the given keyphrase. 285 * It's called at least once with the initial availability.<p/> 286 * 287 * Availability implies whether the hardware on this system is capable of listening for 288 * the given keyphrase or not. <p/> 289 * 290 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 291 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNSUPPORTED 292 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 293 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 294 */ onAvailabilityChanged(int status)295 public abstract void onAvailabilityChanged(int status); 296 /** 297 * Called when the keyphrase is spoken. 298 * This implicitly stops listening for the keyphrase once it's detected. 299 * Clients should start a recognition again once they are done handling this 300 * detection. 301 * 302 * @param eventPayload Payload data for the detection event. 303 * This may contain the trigger audio, if requested when calling 304 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 305 */ onDetected(@onNull EventPayload eventPayload)306 public abstract void onDetected(@NonNull EventPayload eventPayload); 307 /** 308 * Called when the detection fails due to an error. 309 */ onError()310 public abstract void onError(); 311 /** 312 * Called when the recognition is paused temporarily for some reason. 313 * This is an informational callback, and the clients shouldn't be doing anything here 314 * except showing an indication on their UI if they have to. 315 */ onRecognitionPaused()316 public abstract void onRecognitionPaused(); 317 /** 318 * Called when the recognition is resumed after it was temporarily paused. 319 * This is an informational callback, and the clients shouldn't be doing anything here 320 * except showing an indication on their UI if they have to. 321 */ onRecognitionResumed()322 public abstract void onRecognitionResumed(); 323 } 324 325 /** 326 * @param text The keyphrase text to get the detector for. 327 * @param locale The java locale for the detector. 328 * @param callback A non-null Callback for receiving the recognition events. 329 * @param voiceInteractionService The current voice interaction service. 330 * @param modelManagementService A service that allows management of sound models. 331 * 332 * @hide 333 */ AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionService voiceInteractionService, IVoiceInteractionManagerService modelManagementService)334 public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, 335 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 336 IVoiceInteractionService voiceInteractionService, 337 IVoiceInteractionManagerService modelManagementService) { 338 mText = text; 339 mLocale = locale; 340 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 341 mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); 342 mExternalCallback = callback; 343 mHandler = new MyHandler(); 344 mInternalCallback = new SoundTriggerListener(mHandler); 345 mVoiceInteractionService = voiceInteractionService; 346 mModelManagementService = modelManagementService; 347 new RefreshAvailabiltyTask().execute(); 348 } 349 350 /** 351 * Gets the recognition modes supported by the associated keyphrase. 352 * 353 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 354 * @see #RECOGNITION_MODE_VOICE_TRIGGER 355 * 356 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 357 * Callers should only call this method after a supported state callback on 358 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 359 * @throws IllegalStateException if the detector is in an invalid state. 360 * This may happen if another detector has been instantiated or the 361 * {@link VoiceInteractionService} hosting this detector has been shut down. 362 */ getSupportedRecognitionModes()363 public @RecognitionModes int getSupportedRecognitionModes() { 364 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 365 synchronized (mLock) { 366 return getSupportedRecognitionModesLocked(); 367 } 368 } 369 getSupportedRecognitionModesLocked()370 private int getSupportedRecognitionModesLocked() { 371 if (mAvailability == STATE_INVALID) { 372 throw new IllegalStateException( 373 "getSupportedRecognitionModes called on an invalid detector"); 374 } 375 376 // This method only makes sense if we can actually support a recognition. 377 if (mAvailability != STATE_KEYPHRASE_ENROLLED 378 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 379 throw new UnsupportedOperationException( 380 "Getting supported recognition modes for the keyphrase is not supported"); 381 } 382 383 return mKeyphraseMetadata.recognitionModeFlags; 384 } 385 386 /** 387 * Starts recognition for the associated keyphrase. 388 * 389 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 390 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 391 * 392 * @param recognitionFlags The flags to control the recognition properties. 393 * @return Indicates whether the call succeeded or not. 394 * @throws UnsupportedOperationException if the recognition isn't supported. 395 * Callers should only call this method after a supported state callback on 396 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 397 * @throws IllegalStateException if the detector is in an invalid state. 398 * This may happen if another detector has been instantiated or the 399 * {@link VoiceInteractionService} hosting this detector has been shut down. 400 */ startRecognition(@ecognitionFlags int recognitionFlags)401 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 402 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 403 synchronized (mLock) { 404 if (mAvailability == STATE_INVALID) { 405 throw new IllegalStateException("startRecognition called on an invalid detector"); 406 } 407 408 // Check if we can start/stop a recognition. 409 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 410 throw new UnsupportedOperationException( 411 "Recognition for the given keyphrase is not supported"); 412 } 413 414 return startRecognitionLocked(recognitionFlags) == STATUS_OK; 415 } 416 } 417 418 /** 419 * Stops recognition for the associated keyphrase. 420 * 421 * @return Indicates whether the call succeeded or not. 422 * @throws UnsupportedOperationException if the recognition isn't supported. 423 * Callers should only call this method after a supported state callback on 424 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 425 * @throws IllegalStateException if the detector is in an invalid state. 426 * This may happen if another detector has been instantiated or the 427 * {@link VoiceInteractionService} hosting this detector has been shut down. 428 */ stopRecognition()429 public boolean stopRecognition() { 430 if (DBG) Slog.d(TAG, "stopRecognition()"); 431 synchronized (mLock) { 432 if (mAvailability == STATE_INVALID) { 433 throw new IllegalStateException("stopRecognition called on an invalid detector"); 434 } 435 436 // Check if we can start/stop a recognition. 437 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 438 throw new UnsupportedOperationException( 439 "Recognition for the given keyphrase is not supported"); 440 } 441 442 return stopRecognitionLocked() == STATUS_OK; 443 } 444 } 445 446 /** 447 * Creates an intent to start the enrollment for the associated keyphrase. 448 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 449 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 450 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 451 * otherwise {@link #createReEnrollIntent()} should be preferred. 452 * 453 * @return An {@link Intent} to start enrollment for the given keyphrase. 454 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 455 * Callers should only call this method after a supported state callback on 456 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 457 * @throws IllegalStateException if the detector is in an invalid state. 458 * This may happen if another detector has been instantiated or the 459 * {@link VoiceInteractionService} hosting this detector has been shut down. 460 */ createEnrollIntent()461 public Intent createEnrollIntent() { 462 if (DBG) Slog.d(TAG, "createEnrollIntent"); 463 synchronized (mLock) { 464 return getManageIntentLocked(MANAGE_ACTION_ENROLL); 465 } 466 } 467 468 /** 469 * Creates an intent to start the un-enrollment for the associated keyphrase. 470 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 471 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 472 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 473 * 474 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 475 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 476 * Callers should only call this method after a supported state callback on 477 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 478 * @throws IllegalStateException if the detector is in an invalid state. 479 * This may happen if another detector has been instantiated or the 480 * {@link VoiceInteractionService} hosting this detector has been shut down. 481 */ createUnEnrollIntent()482 public Intent createUnEnrollIntent() { 483 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 484 synchronized (mLock) { 485 return getManageIntentLocked(MANAGE_ACTION_UN_ENROLL); 486 } 487 } 488 489 /** 490 * Creates an intent to start the re-enrollment for the associated keyphrase. 491 * This intent must be invoked using {@link Activity#startActivityForResult(Intent, int)}. 492 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 493 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 494 * 495 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 496 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 497 * Callers should only call this method after a supported state callback on 498 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 499 * @throws IllegalStateException if the detector is in an invalid state. 500 * This may happen if another detector has been instantiated or the 501 * {@link VoiceInteractionService} hosting this detector has been shut down. 502 */ createReEnrollIntent()503 public Intent createReEnrollIntent() { 504 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 505 synchronized (mLock) { 506 return getManageIntentLocked(MANAGE_ACTION_RE_ENROLL); 507 } 508 } 509 getManageIntentLocked(int action)510 private Intent getManageIntentLocked(int action) { 511 if (mAvailability == STATE_INVALID) { 512 throw new IllegalStateException("getManageIntent called on an invalid detector"); 513 } 514 515 // This method only makes sense if we can actually support a recognition. 516 if (mAvailability != STATE_KEYPHRASE_ENROLLED 517 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 518 throw new UnsupportedOperationException( 519 "Managing the given keyphrase is not supported"); 520 } 521 522 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 523 } 524 525 /** 526 * Invalidates this hotword detector so that any future calls to this result 527 * in an IllegalStateException. 528 * 529 * @hide 530 */ invalidate()531 void invalidate() { 532 synchronized (mLock) { 533 mAvailability = STATE_INVALID; 534 notifyStateChangedLocked(); 535 } 536 } 537 538 /** 539 * Reloads the sound models from the service. 540 * 541 * @hide 542 */ onSoundModelsChanged()543 void onSoundModelsChanged() { 544 synchronized (mLock) { 545 if (mAvailability == STATE_INVALID 546 || mAvailability == STATE_HARDWARE_UNAVAILABLE 547 || mAvailability == STATE_KEYPHRASE_UNSUPPORTED) { 548 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config"); 549 return; 550 } 551 552 // Stop the recognition before proceeding. 553 // This is done because we want to stop the recognition on an older model if it changed 554 // or was deleted. 555 // The availability change callback should ensure that the client starts recognition 556 // again if needed. 557 stopRecognitionLocked(); 558 559 // Execute a refresh availability task - which should then notify of a change. 560 new RefreshAvailabiltyTask().execute(); 561 } 562 } 563 startRecognitionLocked(int recognitionFlags)564 private int startRecognitionLocked(int recognitionFlags) { 565 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 566 // TODO: Do we need to do something about the confidence level here? 567 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, 568 mKeyphraseMetadata.recognitionModeFlags, 0, new ConfidenceLevel[0]); 569 boolean captureTriggerAudio = 570 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 571 boolean allowMultipleTriggers = 572 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 573 int code = STATUS_ERROR; 574 try { 575 code = mModelManagementService.startRecognition(mVoiceInteractionService, 576 mKeyphraseMetadata.id, mLocale.toLanguageTag(), mInternalCallback, 577 new RecognitionConfig(captureTriggerAudio, allowMultipleTriggers, 578 recognitionExtra, null /* additional data */)); 579 } catch (RemoteException e) { 580 Slog.w(TAG, "RemoteException in startRecognition!", e); 581 } 582 if (code != STATUS_OK) { 583 Slog.w(TAG, "startRecognition() failed with error code " + code); 584 } 585 return code; 586 } 587 stopRecognitionLocked()588 private int stopRecognitionLocked() { 589 int code = STATUS_ERROR; 590 try { 591 code = mModelManagementService.stopRecognition( 592 mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback); 593 } catch (RemoteException e) { 594 Slog.w(TAG, "RemoteException in stopRecognition!", e); 595 } 596 597 if (code != STATUS_OK) { 598 Slog.w(TAG, "stopRecognition() failed with error code " + code); 599 } 600 return code; 601 } 602 notifyStateChangedLocked()603 private void notifyStateChangedLocked() { 604 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 605 message.arg1 = mAvailability; 606 message.sendToTarget(); 607 } 608 609 /** @hide */ 610 static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub { 611 private final Handler mHandler; 612 SoundTriggerListener(Handler handler)613 public SoundTriggerListener(Handler handler) { 614 mHandler = handler; 615 } 616 617 @Override onKeyphraseDetected(KeyphraseRecognitionEvent event)618 public void onKeyphraseDetected(KeyphraseRecognitionEvent event) { 619 if (DBG) { 620 Slog.d(TAG, "onDetected(" + event + ")"); 621 } else { 622 Slog.i(TAG, "onDetected"); 623 } 624 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 625 new EventPayload(event.triggerInData, event.captureAvailable, 626 event.captureFormat, event.captureSession, event.data)) 627 .sendToTarget(); 628 } 629 @Override onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)630 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 631 Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event); 632 } 633 634 @Override onError(int status)635 public void onError(int status) { 636 Slog.i(TAG, "onError: " + status); 637 mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); 638 } 639 640 @Override onRecognitionPaused()641 public void onRecognitionPaused() { 642 Slog.i(TAG, "onRecognitionPaused"); 643 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 644 } 645 646 @Override onRecognitionResumed()647 public void onRecognitionResumed() { 648 Slog.i(TAG, "onRecognitionResumed"); 649 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 650 } 651 } 652 653 class MyHandler extends Handler { 654 @Override handleMessage(Message msg)655 public void handleMessage(Message msg) { 656 synchronized (mLock) { 657 if (mAvailability == STATE_INVALID) { 658 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 659 return; 660 } 661 } 662 663 switch (msg.what) { 664 case MSG_AVAILABILITY_CHANGED: 665 mExternalCallback.onAvailabilityChanged(msg.arg1); 666 break; 667 case MSG_HOTWORD_DETECTED: 668 mExternalCallback.onDetected((EventPayload) msg.obj); 669 break; 670 case MSG_DETECTION_ERROR: 671 mExternalCallback.onError(); 672 break; 673 case MSG_DETECTION_PAUSE: 674 mExternalCallback.onRecognitionPaused(); 675 break; 676 case MSG_DETECTION_RESUME: 677 mExternalCallback.onRecognitionResumed(); 678 break; 679 default: 680 super.handleMessage(msg); 681 } 682 } 683 } 684 685 class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> { 686 687 @Override doInBackground(Void... params)688 public Void doInBackground(Void... params) { 689 int availability = internalGetInitialAvailability(); 690 boolean enrolled = false; 691 // Fetch the sound model if the availability is one of the supported ones. 692 if (availability == STATE_NOT_READY 693 || availability == STATE_KEYPHRASE_UNENROLLED 694 || availability == STATE_KEYPHRASE_ENROLLED) { 695 enrolled = internalGetIsEnrolled(mKeyphraseMetadata.id, mLocale); 696 if (!enrolled) { 697 availability = STATE_KEYPHRASE_UNENROLLED; 698 } else { 699 availability = STATE_KEYPHRASE_ENROLLED; 700 } 701 } 702 703 synchronized (mLock) { 704 if (DBG) { 705 Slog.d(TAG, "Hotword availability changed from " + mAvailability 706 + " -> " + availability); 707 } 708 mAvailability = availability; 709 notifyStateChangedLocked(); 710 } 711 return null; 712 } 713 714 /** 715 * @return The initial availability without checking the enrollment status. 716 */ internalGetInitialAvailability()717 private int internalGetInitialAvailability() { 718 synchronized (mLock) { 719 // This detector has already been invalidated. 720 if (mAvailability == STATE_INVALID) { 721 return STATE_INVALID; 722 } 723 } 724 725 ModuleProperties dspModuleProperties = null; 726 try { 727 dspModuleProperties = 728 mModelManagementService.getDspModuleProperties(mVoiceInteractionService); 729 } catch (RemoteException e) { 730 Slog.w(TAG, "RemoteException in getDspProperties!", e); 731 } 732 // No DSP available 733 if (dspModuleProperties == null) { 734 return STATE_HARDWARE_UNAVAILABLE; 735 } 736 // No enrollment application supports this keyphrase/locale 737 if (mKeyphraseMetadata == null) { 738 return STATE_KEYPHRASE_UNSUPPORTED; 739 } 740 return STATE_NOT_READY; 741 } 742 743 /** 744 * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. 745 */ internalGetIsEnrolled(int keyphraseId, Locale locale)746 private boolean internalGetIsEnrolled(int keyphraseId, Locale locale) { 747 try { 748 return mModelManagementService.isEnrolledForKeyphrase( 749 mVoiceInteractionService, keyphraseId, locale.toLanguageTag()); 750 } catch (RemoteException e) { 751 Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!", e); 752 } 753 return false; 754 } 755 } 756 757 /** @hide */ dump(String prefix, PrintWriter pw)758 public void dump(String prefix, PrintWriter pw) { 759 synchronized (mLock) { 760 pw.print(prefix); pw.print("Text="); pw.println(mText); 761 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 762 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 763 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 764 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 765 } 766 } 767 } 768