1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 22 import android.annotation.IntDef; 23 import android.annotation.NonNull; 24 import android.annotation.Nullable; 25 import android.annotation.RequiresPermission; 26 import android.annotation.SuppressLint; 27 import android.annotation.SystemApi; 28 import android.annotation.TestApi; 29 import android.app.ActivityThread; 30 import android.compat.annotation.UnsupportedAppUsage; 31 import android.content.Context; 32 import android.content.Intent; 33 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 34 import android.hardware.soundtrigger.KeyphraseMetadata; 35 import android.hardware.soundtrigger.SoundTrigger; 36 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 37 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 38 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 39 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 40 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 41 import android.media.AudioFormat; 42 import android.media.permission.Identity; 43 import android.os.AsyncTask; 44 import android.os.Binder; 45 import android.os.Build; 46 import android.os.Handler; 47 import android.os.IBinder; 48 import android.os.Message; 49 import android.os.ParcelFileDescriptor; 50 import android.os.PersistableBundle; 51 import android.os.RemoteException; 52 import android.os.SharedMemory; 53 import android.util.Log; 54 import android.util.Slog; 55 56 import com.android.internal.app.IHotwordRecognitionStatusCallback; 57 import com.android.internal.app.IVoiceInteractionManagerService; 58 import com.android.internal.app.IVoiceInteractionSoundTriggerSession; 59 60 import java.io.PrintWriter; 61 import java.lang.annotation.Retention; 62 import java.lang.annotation.RetentionPolicy; 63 import java.util.Arrays; 64 import java.util.Collections; 65 import java.util.List; 66 import java.util.Locale; 67 68 /** 69 * A class that lets a VoiceInteractionService implementation interact with 70 * always-on keyphrase detection APIs. 71 * 72 * @hide 73 * TODO(b/168605867): Once Metalava supports expressing a removed public, but current system API, 74 * mark and track it as such. 75 */ 76 @SystemApi 77 public class AlwaysOnHotwordDetector extends AbstractHotwordDetector { 78 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 79 /** 80 * Indicates that this hotword detector is no longer valid for any recognition 81 * and should not be used anymore. 82 */ 83 private static final int STATE_INVALID = -3; 84 85 /** 86 * Indicates that recognition for the given keyphrase is not available on the system 87 * because of the hardware configuration. 88 * No further interaction should be performed with the detector that returns this availability. 89 */ 90 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 91 92 /** 93 * Indicates that recognition for the given keyphrase is not supported. 94 * No further interaction should be performed with the detector that returns this availability. 95 * 96 * @deprecated This is no longer a valid state. Enrollment can occur outside of 97 * {@link KeyphraseEnrollmentInfo} through another privileged application. We can no longer 98 * determine ahead of time if the keyphrase and locale are unsupported by the system. 99 */ 100 @Deprecated 101 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 102 103 /** 104 * Indicates that the given keyphrase is not enrolled. 105 * The caller may choose to begin an enrollment flow for the keyphrase. 106 */ 107 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 108 109 /** 110 * Indicates that the given keyphrase is currently enrolled and it's possible to start 111 * recognition for it. 112 */ 113 public static final int STATE_KEYPHRASE_ENROLLED = 2; 114 115 /** 116 * Indicates that the availability state of the active keyphrase can't be known due to an error. 117 * 118 * <p>NOTE: No further interaction should be performed with the detector that returns this 119 * state, it would be better to create {@link AlwaysOnHotwordDetector} again. 120 */ 121 public static final int STATE_ERROR = 3; 122 123 /** 124 * Indicates that the detector isn't ready currently. 125 */ 126 private static final int STATE_NOT_READY = 0; 127 128 //-- Flags for startRecognition ----// 129 /** @hide */ 130 @Retention(RetentionPolicy.SOURCE) 131 @IntDef(flag = true, prefix = { "RECOGNITION_FLAG_" }, value = { 132 RECOGNITION_FLAG_NONE, 133 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 134 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS, 135 RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION, 136 RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION, 137 RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER, 138 }) 139 public @interface RecognitionFlags {} 140 141 /** 142 * Empty flag for {@link #startRecognition(int)}. 143 * 144 * @hide 145 */ 146 public static final int RECOGNITION_FLAG_NONE = 0; 147 148 /** 149 * Recognition flag for {@link #startRecognition(int)} that indicates 150 * whether the trigger audio for hotword needs to be captured. 151 */ 152 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 153 154 /** 155 * Recognition flag for {@link #startRecognition(int)} that indicates 156 * whether the recognition should keep going on even after the keyphrase triggers. 157 * If this flag is specified, it's possible to get multiple triggers after a 158 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 159 * When this isn't specified, the default behavior is to stop recognition once the 160 * keyphrase is spoken, till the caller starts recognition again. 161 */ 162 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 163 164 /** 165 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 166 * if the underlying recognition should use AEC. 167 * This capability may or may not be supported by the system, and support can be queried 168 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 169 * this flag is {@link #AUDIO_CAPABILITY_ECHO_CANCELLATION}. If this flag is passed without the 170 * audio capability supported, there will be no audio effect applied. 171 */ 172 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION = 0x4; 173 174 /** 175 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 176 * if the underlying recognition should use noise suppression. 177 * This capability may or may not be supported by the system, and support can be queried 178 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 179 * this flag is {@link #AUDIO_CAPABILITY_NOISE_SUPPRESSION}. If this flag is passed without the 180 * audio capability supported, there will be no audio effect applied. 181 */ 182 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION = 0x8; 183 184 /** 185 * Recognition flag for {@link #startRecognition(int)} that indicates whether the recognition 186 * should continue after battery saver mode is enabled. 187 * When this flag is specified, the caller will be checked for 188 * {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER} permission granted. 189 */ 190 public static final int RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER = 0x10; 191 192 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 193 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 194 195 /** @hide */ 196 @Retention(RetentionPolicy.SOURCE) 197 @IntDef(flag = true, prefix = { "RECOGNITION_MODE_" }, value = { 198 RECOGNITION_MODE_VOICE_TRIGGER, 199 RECOGNITION_MODE_USER_IDENTIFICATION, 200 }) 201 public @interface RecognitionModes {} 202 203 /** 204 * Simple recognition of the key phrase. 205 * Returned by {@link #getSupportedRecognitionModes()} 206 */ 207 public static final int RECOGNITION_MODE_VOICE_TRIGGER 208 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 209 210 /** 211 * User identification performed with the keyphrase recognition. 212 * Returned by {@link #getSupportedRecognitionModes()} 213 */ 214 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 215 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 216 217 //-- Audio capabilities. Values in returned bit field for getSupportedAudioCapabilities() --// 218 219 /** @hide */ 220 @Retention(RetentionPolicy.SOURCE) 221 @IntDef(flag = true, prefix = { "AUDIO_CAPABILITY_" }, value = { 222 AUDIO_CAPABILITY_ECHO_CANCELLATION, 223 AUDIO_CAPABILITY_NOISE_SUPPRESSION, 224 }) 225 public @interface AudioCapabilities {} 226 227 /** 228 * If set the underlying module supports AEC. 229 * Returned by {@link #getSupportedAudioCapabilities()} 230 */ 231 public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION = 232 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_ECHO_CANCELLATION; 233 234 /** 235 * If set, the underlying module supports noise suppression. 236 * Returned by {@link #getSupportedAudioCapabilities()} 237 */ 238 public static final int AUDIO_CAPABILITY_NOISE_SUPPRESSION = 239 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_NOISE_SUPPRESSION; 240 241 /** @hide */ 242 @Retention(RetentionPolicy.SOURCE) 243 @IntDef(flag = true, prefix = { "MODEL_PARAM_" }, value = { 244 MODEL_PARAM_THRESHOLD_FACTOR, 245 }) 246 public @interface ModelParams {} 247 248 /** 249 * Controls the sensitivity threshold adjustment factor for a given model. 250 * Negative value corresponds to less sensitive model (high threshold) and 251 * a positive value corresponds to a more sensitive model (low threshold). 252 * Default value is 0. 253 */ 254 public static final int MODEL_PARAM_THRESHOLD_FACTOR = 255 android.hardware.soundtrigger.ModelParams.THRESHOLD_FACTOR; 256 257 static final String TAG = "AlwaysOnHotwordDetector"; 258 static final boolean DBG = false; 259 260 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 261 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 262 263 private static final int MSG_AVAILABILITY_CHANGED = 1; 264 private static final int MSG_HOTWORD_DETECTED = 2; 265 private static final int MSG_DETECTION_ERROR = 3; 266 private static final int MSG_DETECTION_PAUSE = 4; 267 private static final int MSG_DETECTION_RESUME = 5; 268 private static final int MSG_HOTWORD_REJECTED = 6; 269 private static final int MSG_HOTWORD_STATUS_REPORTED = 7; 270 private static final int MSG_PROCESS_RESTARTED = 8; 271 272 private final String mText; 273 private final Locale mLocale; 274 /** 275 * The metadata of the Keyphrase, derived from the enrollment application. 276 * This may be null if this keyphrase isn't supported by the enrollment application. 277 */ 278 @Nullable 279 private KeyphraseMetadata mKeyphraseMetadata; 280 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 281 private final IVoiceInteractionManagerService mModelManagementService; 282 private final IVoiceInteractionSoundTriggerSession mSoundTriggerSession; 283 private final SoundTriggerListener mInternalCallback; 284 private final Callback mExternalCallback; 285 private final Handler mHandler; 286 private final IBinder mBinder = new Binder(); 287 private final int mTargetSdkVersion; 288 private final boolean mSupportHotwordDetectionService; 289 290 private int mAvailability = STATE_NOT_READY; 291 292 /** 293 * A ModelParamRange is a representation of supported parameter range for a 294 * given loaded model. 295 */ 296 public static final class ModelParamRange { 297 private final SoundTrigger.ModelParamRange mModelParamRange; 298 299 /** @hide */ ModelParamRange(SoundTrigger.ModelParamRange modelParamRange)300 ModelParamRange(SoundTrigger.ModelParamRange modelParamRange) { 301 mModelParamRange = modelParamRange; 302 } 303 304 /** 305 * Get the beginning of the param range 306 * 307 * @return The inclusive start of the supported range. 308 */ getStart()309 public int getStart() { 310 return mModelParamRange.getStart(); 311 } 312 313 /** 314 * Get the end of the param range 315 * 316 * @return The inclusive end of the supported range. 317 */ getEnd()318 public int getEnd() { 319 return mModelParamRange.getEnd(); 320 } 321 322 @Override 323 @NonNull toString()324 public String toString() { 325 return mModelParamRange.toString(); 326 } 327 328 @Override equals(@ullable Object obj)329 public boolean equals(@Nullable Object obj) { 330 return mModelParamRange.equals(obj); 331 } 332 333 @Override hashCode()334 public int hashCode() { 335 return mModelParamRange.hashCode(); 336 } 337 } 338 339 /** 340 * Additional payload for {@link Callback#onDetected}. 341 */ 342 public static class EventPayload { 343 344 /** 345 * Flags for describing the data format provided in the event payload. 346 * 347 * @hide 348 */ 349 @Retention(RetentionPolicy.SOURCE) 350 @IntDef(prefix = {"DATA_FORMAT_"}, value = { 351 DATA_FORMAT_RAW, 352 DATA_FORMAT_TRIGGER_AUDIO, 353 }) 354 public @interface DataFormat { 355 } 356 357 /** 358 * Data format is not strictly defined by the framework, and the 359 * {@link android.hardware.soundtrigger.SoundTriggerModule} voice engine may populate this 360 * field in any format. 361 */ 362 public static final int DATA_FORMAT_RAW = 0; 363 364 /** 365 * Data format is defined as trigger audio. 366 * 367 * <p>When this format is used, {@link #getCaptureAudioFormat()} can be used to understand 368 * further the audio format for reading the data. 369 * 370 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 371 */ 372 public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; 373 374 @DataFormat 375 private final int mDataFormat; 376 // Indicates if {@code captureSession} can be used to continue capturing more audio 377 // from the DSP hardware. 378 private final boolean mCaptureAvailable; 379 // The session to use when attempting to capture more audio from the DSP hardware. 380 private final int mCaptureSession; 381 private final AudioFormat mAudioFormat; 382 // Raw data associated with the event. 383 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 384 private final byte[] mData; 385 private final HotwordDetectedResult mHotwordDetectedResult; 386 private final ParcelFileDescriptor mAudioStream; 387 private final List<KeyphraseRecognitionExtra> mKephraseExtras; 388 EventPayload(boolean captureAvailable, @Nullable AudioFormat audioFormat, int captureSession, @DataFormat int dataFormat, @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, @Nullable ParcelFileDescriptor audioStream, @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras)389 private EventPayload(boolean captureAvailable, 390 @Nullable AudioFormat audioFormat, 391 int captureSession, 392 @DataFormat int dataFormat, 393 @Nullable byte[] data, 394 @Nullable HotwordDetectedResult hotwordDetectedResult, 395 @Nullable ParcelFileDescriptor audioStream, 396 @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras) { 397 mCaptureAvailable = captureAvailable; 398 mCaptureSession = captureSession; 399 mAudioFormat = audioFormat; 400 mDataFormat = dataFormat; 401 mData = data; 402 mHotwordDetectedResult = hotwordDetectedResult; 403 mAudioStream = audioStream; 404 mKephraseExtras = keyphraseExtras; 405 } 406 407 /** 408 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 409 * May be null if there's no audio present. 410 */ 411 @Nullable getCaptureAudioFormat()412 public AudioFormat getCaptureAudioFormat() { 413 return mAudioFormat; 414 } 415 416 /** 417 * Gets the raw audio that triggered the keyphrase. 418 * This may be null if the trigger audio isn't available. 419 * If non-null, the format of the audio can be obtained by calling 420 * {@link #getCaptureAudioFormat()}. 421 * 422 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 423 * @deprecated Use {@link #getData()} instead. 424 */ 425 @Deprecated 426 @Nullable getTriggerAudio()427 public byte[] getTriggerAudio() { 428 if (mDataFormat == DATA_FORMAT_TRIGGER_AUDIO) { 429 return mData; 430 } else { 431 return null; 432 } 433 } 434 435 /** 436 * Conveys the format of the additional data that is triggered with the keyphrase event. 437 * 438 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 439 * @see DataFormat 440 */ 441 @DataFormat getDataFormat()442 public int getDataFormat() { 443 return mDataFormat; 444 } 445 446 /** 447 * Gets additional raw data that is triggered with the keyphrase event. 448 * 449 * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this 450 * field with opaque data for use by system applications who know about voice 451 * engine internals. Data may be null if the field is not populated by the 452 * {@link android.hardware.soundtrigger.SoundTriggerModule}. 453 * 454 * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the 455 * entirety of this buffer is expected to be of the format from 456 * {@link #getCaptureAudioFormat()}. 457 * 458 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 459 */ 460 @Nullable getData()461 public byte[] getData() { 462 return mData; 463 } 464 465 /** 466 * Gets the session ID to start a capture from the DSP. 467 * This may be null if streaming capture isn't possible. 468 * If non-null, the format of the audio that can be captured can be 469 * obtained using {@link #getCaptureAudioFormat()}. 470 * 471 * TODO: Candidate for Public API when the API to start capture with a session ID 472 * is made public. 473 * 474 * TODO: Add this to {@link #getCaptureAudioFormat()}: 475 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 476 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 477 * for either the trigger or a streaming session." 478 * 479 * TODO: Should this return a known invalid value instead? 480 * 481 * @hide 482 */ 483 @Nullable 484 @UnsupportedAppUsage getCaptureSession()485 public Integer getCaptureSession() { 486 if (mCaptureAvailable) { 487 return mCaptureSession; 488 } else { 489 return null; 490 } 491 } 492 493 /** 494 * Returns {@link HotwordDetectedResult} associated with the hotword event, passed from 495 * {@link HotwordDetectionService}. 496 */ 497 @Nullable getHotwordDetectedResult()498 public HotwordDetectedResult getHotwordDetectedResult() { 499 return mHotwordDetectedResult; 500 } 501 502 /** 503 * Returns a stream with bytes corresponding to the open audio stream with hotword data. 504 * 505 * <p>This data represents an audio stream in the format returned by 506 * {@link #getCaptureAudioFormat}. 507 * 508 * <p>Clients are expected to start consuming the stream within 1 second of receiving the 509 * event. 510 * 511 * <p>When this method returns a non-null, clients must close this stream when it's no 512 * longer needed. Failing to do so will result in microphone being open for longer periods 513 * of time, and app being attributed for microphone usage. 514 */ 515 @Nullable getAudioStream()516 public ParcelFileDescriptor getAudioStream() { 517 return mAudioStream; 518 } 519 520 /** 521 * Returns the keyphrases recognized by the voice engine with additional confidence 522 * information 523 * 524 * @return List of keyphrase extras describing additional data for each keyphrase the voice 525 * engine triggered on for this event. The ordering of the list is preserved based on what 526 * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. 527 */ 528 @NonNull getKeyphraseRecognitionExtras()529 public List<KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras() { 530 return mKephraseExtras; 531 } 532 533 /** 534 * Builder class for {@link EventPayload} objects 535 * 536 * @hide 537 */ 538 @TestApi 539 public static final class Builder { 540 private boolean mCaptureAvailable = false; 541 private int mCaptureSession = -1; 542 private AudioFormat mAudioFormat = null; 543 @DataFormat 544 private int mDataFormat = DATA_FORMAT_RAW; 545 private byte[] mData = null; 546 private HotwordDetectedResult mHotwordDetectedResult = null; 547 private ParcelFileDescriptor mAudioStream = null; 548 private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList(); 549 Builder()550 public Builder() {} 551 Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent)552 Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { 553 setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); 554 setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); 555 if (keyphraseRecognitionEvent.getCaptureFormat() != null) { 556 setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); 557 } 558 setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO 559 : DATA_FORMAT_RAW); 560 if (keyphraseRecognitionEvent.getData() != null) { 561 setData(keyphraseRecognitionEvent.getData()); 562 } 563 if (keyphraseRecognitionEvent.keyphraseExtras != null) { 564 setKeyphraseRecognitionExtras( 565 Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); 566 } 567 } 568 569 /** 570 * Indicates if {@code captureSession} can be used to continue capturing more audio from 571 * the DSP hardware. 572 */ 573 @SuppressLint("MissingGetterMatchingBuilder") 574 @NonNull setCaptureAvailable(boolean captureAvailable)575 public Builder setCaptureAvailable(boolean captureAvailable) { 576 mCaptureAvailable = captureAvailable; 577 return this; 578 } 579 580 /** 581 * Sets the session ID to start a capture from the DSP. 582 */ 583 @SuppressLint("MissingGetterMatchingBuilder") 584 @NonNull setCaptureSession(int captureSession)585 public Builder setCaptureSession(int captureSession) { 586 mCaptureSession = captureSession; 587 return this; 588 } 589 590 /** 591 * Sets the format of the audio obtained using {@link #getTriggerAudio()}. 592 */ 593 @NonNull setCaptureAudioFormat(@onNull AudioFormat audioFormat)594 public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { 595 mAudioFormat = audioFormat; 596 return this; 597 } 598 599 /** 600 * Conveys the format of the additional data that is triggered with the keyphrase event. 601 * 602 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 603 * @see DataFormat 604 */ 605 @NonNull setDataFormat(@ataFormat int dataFormat)606 public Builder setDataFormat(@DataFormat int dataFormat) { 607 mDataFormat = dataFormat; 608 return this; 609 } 610 611 /** 612 * Sets additional raw data that is triggered with the keyphrase event. 613 * 614 * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this 615 * field with opaque data for use by system applications who know about voice 616 * engine internals. Data may be null if the field is not populated by the 617 * {@link android.hardware.soundtrigger.SoundTriggerModule}. 618 * 619 * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the 620 * entirety of this 621 * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. 622 * 623 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 624 */ 625 @NonNull setData(@onNull byte[] data)626 public Builder setData(@NonNull byte[] data) { 627 mData = data; 628 return this; 629 } 630 631 /** 632 * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from 633 * {@link HotwordDetectionService}. 634 */ 635 @NonNull setHotwordDetectedResult( @onNull HotwordDetectedResult hotwordDetectedResult)636 public Builder setHotwordDetectedResult( 637 @NonNull HotwordDetectedResult hotwordDetectedResult) { 638 mHotwordDetectedResult = hotwordDetectedResult; 639 return this; 640 } 641 642 /** 643 * Sets a stream with bytes corresponding to the open audio stream with hotword data. 644 * 645 * <p>This data represents an audio stream in the format returned by 646 * {@link #getCaptureAudioFormat}. 647 * 648 * <p>Clients are expected to start consuming the stream within 1 second of receiving 649 * the 650 * event. 651 */ 652 @NonNull setAudioStream(@onNull ParcelFileDescriptor audioStream)653 public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { 654 mAudioStream = audioStream; 655 return this; 656 } 657 658 /** 659 * Sets the keyphrases recognized by the voice engine with additional confidence 660 * information 661 */ 662 @NonNull setKeyphraseRecognitionExtras( @onNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)663 public Builder setKeyphraseRecognitionExtras( 664 @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) { 665 mKeyphraseExtras = keyphraseRecognitionExtras; 666 return this; 667 } 668 669 /** 670 * Builds an {@link EventPayload} instance 671 */ 672 @NonNull build()673 public EventPayload build() { 674 return new EventPayload(mCaptureAvailable, mAudioFormat, mCaptureSession, 675 mDataFormat, mData, mHotwordDetectedResult, mAudioStream, 676 mKeyphraseExtras); 677 } 678 } 679 } 680 681 /** 682 * Callbacks for always-on hotword detection. 683 */ 684 public abstract static class Callback implements HotwordDetector.Callback { 685 686 /** 687 * Updates the availability state of the active keyphrase and locale on every keyphrase 688 * sound model change. 689 * 690 * <p>This API is called whenever there's a possibility that the keyphrase associated 691 * with this detector has been updated. It is not guaranteed that there is in fact any 692 * change, as it may be called for other reasons.</p> 693 * 694 * <p>This API is also guaranteed to be called right after an AlwaysOnHotwordDetector 695 * instance is created to updated the current availability state.</p> 696 * 697 * <p>Availability implies the current enrollment state of the given keyphrase. If the 698 * hardware on this system is not capable of listening for the given keyphrase, 699 * {@link AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE} will be returned. 700 * 701 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 702 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 703 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 704 * @see AlwaysOnHotwordDetector#STATE_ERROR 705 */ onAvailabilityChanged(int status)706 public abstract void onAvailabilityChanged(int status); 707 708 /** 709 * Called when the keyphrase is spoken. 710 * This implicitly stops listening for the keyphrase once it's detected. 711 * Clients should start a recognition again once they are done handling this 712 * detection. 713 * 714 * @param eventPayload Payload data for the detection event. 715 * This may contain the trigger audio, if requested when calling 716 * {@link AlwaysOnHotwordDetector#startRecognition(int)}. 717 */ onDetected(@onNull EventPayload eventPayload)718 public abstract void onDetected(@NonNull EventPayload eventPayload); 719 720 /** 721 * Called when the detection fails due to an error. 722 */ onError()723 public abstract void onError(); 724 725 /** 726 * Called when the recognition is paused temporarily for some reason. 727 * This is an informational callback, and the clients shouldn't be doing anything here 728 * except showing an indication on their UI if they have to. 729 */ onRecognitionPaused()730 public abstract void onRecognitionPaused(); 731 732 /** 733 * Called when the recognition is resumed after it was temporarily paused. 734 * This is an informational callback, and the clients shouldn't be doing anything here 735 * except showing an indication on their UI if they have to. 736 */ onRecognitionResumed()737 public abstract void onRecognitionResumed(); 738 739 /** 740 * Called when the {@link HotwordDetectionService second stage detection} did not detect the 741 * keyphrase. 742 * 743 * @param result Info about the second stage detection result, provided by the 744 * {@link HotwordDetectionService}. 745 */ onRejected(@onNull HotwordRejectedResult result)746 public void onRejected(@NonNull HotwordRejectedResult result) { 747 } 748 749 /** 750 * Called when the {@link HotwordDetectionService} is created by the system and given a 751 * short amount of time to report it's initialization state. 752 * 753 * @param status Info about initialization state of {@link HotwordDetectionService}; the 754 * allowed values are {@link HotwordDetectionService#INITIALIZATION_STATUS_SUCCESS}, 755 * 1<->{@link HotwordDetectionService#getMaxCustomInitializationStatus()}, 756 * {@link HotwordDetectionService#INITIALIZATION_STATUS_UNKNOWN}. 757 */ onHotwordDetectionServiceInitialized(int status)758 public void onHotwordDetectionServiceInitialized(int status) { 759 } 760 761 /** 762 * Called with the {@link HotwordDetectionService} is restarted. 763 * 764 * Clients are expected to call {@link HotwordDetector#updateState} to share the state with 765 * the newly created service. 766 */ onHotwordDetectionServiceRestarted()767 public void onHotwordDetectionServiceRestarted() { 768 } 769 } 770 771 /** 772 * @param text The keyphrase text to get the detector for. 773 * @param locale The java locale for the detector. 774 * @param callback A non-null Callback for receiving the recognition events. 775 * @param modelManagementService A service that allows management of sound models. 776 * @param targetSdkVersion The target SDK version. 777 * @param supportHotwordDetectionService {@code true} if hotword detection service should be 778 * triggered, otherwise {@code false}. 779 * @param options Application configuration data provided by the 780 * {@link VoiceInteractionService}. PersistableBundle does not allow any remotable objects or 781 * other contents that can be used to communicate with other processes. 782 * @param sharedMemory The unrestricted data blob provided by the 783 * {@link VoiceInteractionService}. Use this to provide the hotword models data or other 784 * such data to the trusted process. 785 * 786 * @hide 787 */ AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, boolean supportHotwordDetectionService, @Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory)788 public AlwaysOnHotwordDetector(String text, Locale locale, Callback callback, 789 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 790 IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, 791 boolean supportHotwordDetectionService, @Nullable PersistableBundle options, 792 @Nullable SharedMemory sharedMemory) { 793 super(modelManagementService, callback, 794 supportHotwordDetectionService ? DETECTOR_TYPE_TRUSTED_HOTWORD_DSP 795 : DETECTOR_TYPE_NORMAL); 796 797 mHandler = new MyHandler(); 798 mText = text; 799 mLocale = locale; 800 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 801 mExternalCallback = callback; 802 mInternalCallback = new SoundTriggerListener(mHandler); 803 mModelManagementService = modelManagementService; 804 mTargetSdkVersion = targetSdkVersion; 805 mSupportHotwordDetectionService = supportHotwordDetectionService; 806 if (mSupportHotwordDetectionService) { 807 updateStateLocked(options, sharedMemory, mInternalCallback, 808 DETECTOR_TYPE_TRUSTED_HOTWORD_DSP); 809 } 810 try { 811 Identity identity = new Identity(); 812 identity.packageName = ActivityThread.currentOpPackageName(); 813 mSoundTriggerSession = mModelManagementService.createSoundTriggerSessionAsOriginator( 814 identity, mBinder); 815 } catch (RemoteException e) { 816 throw e.rethrowAsRuntimeException(); 817 } 818 new RefreshAvailabiltyTask().execute(); 819 } 820 821 /** 822 * {@inheritDoc} 823 * 824 * @throws IllegalStateException if this AlwaysOnHotwordDetector wasn't specified to use a 825 * {@link HotwordDetectionService} when it was created. In addition, if this 826 * AlwaysOnHotwordDetector is in an invalid or error state. 827 */ 828 @Override updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)829 public final void updateState(@Nullable PersistableBundle options, 830 @Nullable SharedMemory sharedMemory) { 831 synchronized (mLock) { 832 if (!mSupportHotwordDetectionService) { 833 throw new IllegalStateException( 834 "updateState called, but it doesn't support hotword detection service"); 835 } 836 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 837 throw new IllegalStateException( 838 "updateState called on an invalid detector or error state"); 839 } 840 } 841 842 super.updateState(options, sharedMemory); 843 } 844 845 /** 846 * Test API to simulate to trigger hardware recognition event for test. 847 * 848 * @hide 849 */ 850 @TestApi 851 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)852 public void triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, 853 boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, 854 boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, 855 @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) { 856 Log.d(TAG, "triggerHardwareRecognitionEventForTest()"); 857 synchronized (mLock) { 858 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 859 throw new IllegalStateException("triggerHardwareRecognitionEventForTest called on" 860 + " an invalid detector or error state"); 861 } 862 try { 863 mModelManagementService.triggerHardwareRecognitionEventForTest( 864 new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable, 865 captureSession, captureDelayMs, capturePreambleMs, triggerInData, 866 captureFormat, data, keyphraseRecognitionExtras.toArray( 867 new KeyphraseRecognitionExtra[0])), 868 mInternalCallback); 869 } catch (RemoteException e) { 870 throw e.rethrowFromSystemServer(); 871 } 872 } 873 } 874 875 /** 876 * Gets the recognition modes supported by the associated keyphrase. 877 * 878 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 879 * @see #RECOGNITION_MODE_VOICE_TRIGGER 880 * 881 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 882 * Callers should only call this method after a supported state callback on 883 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 884 * @throws IllegalStateException if the detector is in an invalid or error state. 885 * This may happen if another detector has been instantiated or the 886 * {@link VoiceInteractionService} hosting this detector has been shut down. 887 */ getSupportedRecognitionModes()888 public @RecognitionModes int getSupportedRecognitionModes() { 889 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 890 synchronized (mLock) { 891 return getSupportedRecognitionModesLocked(); 892 } 893 } 894 getSupportedRecognitionModesLocked()895 private int getSupportedRecognitionModesLocked() { 896 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 897 throw new IllegalStateException( 898 "getSupportedRecognitionModes called on an invalid detector or error state"); 899 } 900 901 // This method only makes sense if we can actually support a recognition. 902 if (mAvailability != STATE_KEYPHRASE_ENROLLED || mKeyphraseMetadata == null) { 903 throw new UnsupportedOperationException( 904 "Getting supported recognition modes for the keyphrase is not supported"); 905 } 906 907 return mKeyphraseMetadata.getRecognitionModeFlags(); 908 } 909 910 /** 911 * Get the audio capabilities supported by the platform which can be enabled when 912 * starting a recognition. 913 * Caller must be the active voice interaction service via 914 * Settings.Secure.VOICE_INTERACTION_SERVICE. 915 * 916 * @see #AUDIO_CAPABILITY_ECHO_CANCELLATION 917 * @see #AUDIO_CAPABILITY_NOISE_SUPPRESSION 918 * 919 * @return Bit field encoding of the AudioCapabilities supported. 920 */ 921 @AudioCapabilities getSupportedAudioCapabilities()922 public int getSupportedAudioCapabilities() { 923 if (DBG) Slog.d(TAG, "getSupportedAudioCapabilities()"); 924 synchronized (mLock) { 925 return getSupportedAudioCapabilitiesLocked(); 926 } 927 } 928 getSupportedAudioCapabilitiesLocked()929 private int getSupportedAudioCapabilitiesLocked() { 930 try { 931 ModuleProperties properties = 932 mSoundTriggerSession.getDspModuleProperties(); 933 if (properties != null) { 934 return properties.getAudioCapabilities(); 935 } 936 937 return 0; 938 } catch (RemoteException e) { 939 throw e.rethrowFromSystemServer(); 940 } 941 } 942 943 /** 944 * Starts recognition for the associated keyphrase. 945 * Caller must be the active voice interaction service via 946 * Settings.Secure.VOICE_INTERACTION_SERVICE. 947 * 948 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 949 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 950 * 951 * @param recognitionFlags The flags to control the recognition properties. 952 * @return Indicates whether the call succeeded or not. 953 * @throws UnsupportedOperationException if the recognition isn't supported. 954 * Callers should only call this method after a supported state callback on 955 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 956 * @throws IllegalStateException if the detector is in an invalid or error state. 957 * This may happen if another detector has been instantiated or the 958 * {@link VoiceInteractionService} hosting this detector has been shut down. 959 */ 960 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) startRecognition(@ecognitionFlags int recognitionFlags)961 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 962 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 963 synchronized (mLock) { 964 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 965 throw new IllegalStateException( 966 "startRecognition called on an invalid detector or error state"); 967 } 968 969 // Check if we can start/stop a recognition. 970 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 971 throw new UnsupportedOperationException( 972 "Recognition for the given keyphrase is not supported"); 973 } 974 975 return startRecognitionLocked(recognitionFlags) == STATUS_OK; 976 } 977 } 978 979 /** 980 * Starts recognition for the associated keyphrase. 981 * 982 * @see #startRecognition(int) 983 */ 984 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 985 @Override startRecognition()986 public boolean startRecognition() { 987 return startRecognition(0); 988 } 989 990 /** 991 * Stops recognition for the associated keyphrase. 992 * Caller must be the active voice interaction service via 993 * Settings.Secure.VOICE_INTERACTION_SERVICE. 994 * 995 * @return Indicates whether the call succeeded or not. 996 * @throws UnsupportedOperationException if the recognition isn't supported. 997 * Callers should only call this method after a supported state callback on 998 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 999 * @throws IllegalStateException if the detector is in an invalid or error state. 1000 * This may happen if another detector has been instantiated or the 1001 * {@link VoiceInteractionService} hosting this detector has been shut down. 1002 */ 1003 // TODO: Remove this RequiresPermission since it isn't actually enforced. Also fix the javadoc 1004 // about permissions enforcement (when it throws vs when it just returns false) for other 1005 // methods in this class. 1006 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 1007 @Override stopRecognition()1008 public boolean stopRecognition() { 1009 if (DBG) Slog.d(TAG, "stopRecognition()"); 1010 synchronized (mLock) { 1011 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1012 throw new IllegalStateException( 1013 "stopRecognition called on an invalid detector or error state"); 1014 } 1015 1016 // Check if we can start/stop a recognition. 1017 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 1018 throw new UnsupportedOperationException( 1019 "Recognition for the given keyphrase is not supported"); 1020 } 1021 1022 return stopRecognitionLocked() == STATUS_OK; 1023 } 1024 } 1025 1026 /** 1027 * Set a model specific {@link ModelParams} with the given value. This 1028 * parameter will keep its value for the duration the model is loaded regardless of starting and 1029 * stopping recognition. Once the model is unloaded, the value will be lost. 1030 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before calling this 1031 * method. 1032 * Caller must be the active voice interaction service via 1033 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1034 * 1035 * @param modelParam {@link ModelParams} 1036 * @param value Value to set 1037 * @return - {@link SoundTrigger#STATUS_OK} in case of success 1038 * - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached 1039 * - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter 1040 * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or 1041 * if API is not supported by HAL 1042 * @throws IllegalStateException if the detector is in an invalid or error state. 1043 * This may happen if another detector has been instantiated or the 1044 * {@link VoiceInteractionService} hosting this detector has been shut down. 1045 */ 1046 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) setParameter(@odelParams int modelParam, int value)1047 public int setParameter(@ModelParams int modelParam, int value) { 1048 if (DBG) { 1049 Slog.d(TAG, "setParameter(" + modelParam + ", " + value + ")"); 1050 } 1051 1052 synchronized (mLock) { 1053 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1054 throw new IllegalStateException( 1055 "setParameter called on an invalid detector or error state"); 1056 } 1057 1058 return setParameterLocked(modelParam, value); 1059 } 1060 } 1061 1062 /** 1063 * Get a model specific {@link ModelParams}. This parameter will keep its value 1064 * for the duration the model is loaded regardless of starting and stopping recognition. 1065 * Once the model is unloaded, the value will be lost. If the value is not set, a default 1066 * value is returned. See {@link ModelParams} for parameter default values. 1067 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before 1068 * calling this method. 1069 * Caller must be the active voice interaction service via 1070 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1071 * 1072 * @param modelParam {@link ModelParams} 1073 * @return value of parameter 1074 * @throws IllegalStateException if the detector is in an invalid or error state. 1075 * This may happen if another detector has been instantiated or the 1076 * {@link VoiceInteractionService} hosting this detector has been shut down. 1077 */ 1078 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) getParameter(@odelParams int modelParam)1079 public int getParameter(@ModelParams int modelParam) { 1080 if (DBG) { 1081 Slog.d(TAG, "getParameter(" + modelParam + ")"); 1082 } 1083 1084 synchronized (mLock) { 1085 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1086 throw new IllegalStateException( 1087 "getParameter called on an invalid detector or error state"); 1088 } 1089 1090 return getParameterLocked(modelParam); 1091 } 1092 } 1093 1094 /** 1095 * Determine if parameter control is supported for the given model handle. 1096 * This method should be checked prior to calling {@link AlwaysOnHotwordDetector#setParameter} 1097 * or {@link AlwaysOnHotwordDetector#getParameter}. 1098 * Caller must be the active voice interaction service via 1099 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1100 * 1101 * @param modelParam {@link ModelParams} 1102 * @return supported range of parameter, null if not supported 1103 * @throws IllegalStateException if the detector is in an invalid or error state. 1104 * This may happen if another detector has been instantiated or the 1105 * {@link VoiceInteractionService} hosting this detector has been shut down. 1106 */ 1107 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 1108 @Nullable queryParameter(@odelParams int modelParam)1109 public ModelParamRange queryParameter(@ModelParams int modelParam) { 1110 if (DBG) { 1111 Slog.d(TAG, "queryParameter(" + modelParam + ")"); 1112 } 1113 1114 synchronized (mLock) { 1115 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1116 throw new IllegalStateException( 1117 "queryParameter called on an invalid detector or error state"); 1118 } 1119 1120 return queryParameterLocked(modelParam); 1121 } 1122 } 1123 1124 /** 1125 * Creates an intent to start the enrollment for the associated keyphrase. 1126 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1127 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 1128 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 1129 * otherwise {@link #createReEnrollIntent()} should be preferred. 1130 * 1131 * @return An {@link Intent} to start enrollment for the given keyphrase. 1132 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1133 * Callers should only call this method after a supported state callback on 1134 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1135 * @throws IllegalStateException if the detector is in an invalid state. 1136 * This may happen if another detector has been instantiated or the 1137 * {@link VoiceInteractionService} hosting this detector has been shut down. 1138 */ 1139 @Nullable createEnrollIntent()1140 public Intent createEnrollIntent() { 1141 if (DBG) Slog.d(TAG, "createEnrollIntent"); 1142 synchronized (mLock) { 1143 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_ENROLL); 1144 } 1145 } 1146 1147 /** 1148 * Creates an intent to start the un-enrollment for the associated keyphrase. 1149 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1150 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 1151 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 1152 * 1153 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 1154 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1155 * Callers should only call this method after a supported state callback on 1156 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1157 * @throws IllegalStateException if the detector is in an invalid state. 1158 * This may happen if another detector has been instantiated or the 1159 * {@link VoiceInteractionService} hosting this detector has been shut down. 1160 */ 1161 @Nullable createUnEnrollIntent()1162 public Intent createUnEnrollIntent() { 1163 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 1164 synchronized (mLock) { 1165 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_UN_ENROLL); 1166 } 1167 } 1168 1169 /** 1170 * Creates an intent to start the re-enrollment for the associated keyphrase. 1171 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1172 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 1173 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 1174 * 1175 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 1176 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1177 * Callers should only call this method after a supported state callback on 1178 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1179 * @throws IllegalStateException if the detector is in an invalid or error state. 1180 * This may happen if another detector has been instantiated or the 1181 * {@link VoiceInteractionService} hosting this detector has been shut down. 1182 */ 1183 @Nullable createReEnrollIntent()1184 public Intent createReEnrollIntent() { 1185 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 1186 synchronized (mLock) { 1187 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_RE_ENROLL); 1188 } 1189 } 1190 getManageIntentLocked(@eyphraseEnrollmentInfo.ManageActions int action)1191 private Intent getManageIntentLocked(@KeyphraseEnrollmentInfo.ManageActions int action) { 1192 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1193 throw new IllegalStateException( 1194 "getManageIntent called on an invalid detector or error state"); 1195 } 1196 1197 // This method only makes sense if we can actually support a recognition. 1198 if (mAvailability != STATE_KEYPHRASE_ENROLLED 1199 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 1200 throw new UnsupportedOperationException( 1201 "Managing the given keyphrase is not supported"); 1202 } 1203 1204 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 1205 } 1206 1207 /** 1208 * Invalidates this hotword detector so that any future calls to this result 1209 * in an IllegalStateException. 1210 */ 1211 @Override destroy()1212 public void destroy() { 1213 synchronized (mLock) { 1214 if (mAvailability == STATE_KEYPHRASE_ENROLLED) { 1215 stopRecognition(); 1216 } 1217 1218 mAvailability = STATE_INVALID; 1219 notifyStateChangedLocked(); 1220 1221 if (mSupportHotwordDetectionService) { 1222 try { 1223 mModelManagementService.shutdownHotwordDetectionService(); 1224 } catch (RemoteException e) { 1225 throw e.rethrowFromSystemServer(); 1226 } 1227 } 1228 } 1229 super.destroy(); 1230 } 1231 1232 /** 1233 * Reloads the sound models from the service. 1234 * 1235 * @hide 1236 */ onSoundModelsChanged()1237 void onSoundModelsChanged() { 1238 synchronized (mLock) { 1239 if (mAvailability == STATE_INVALID 1240 || mAvailability == STATE_HARDWARE_UNAVAILABLE 1241 || mAvailability == STATE_ERROR) { 1242 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config" 1243 + " or in the error state"); 1244 return; 1245 } 1246 1247 // Stop the recognition before proceeding. 1248 // This is done because we want to stop the recognition on an older model if it changed 1249 // or was deleted. 1250 // The availability change callback should ensure that the client starts recognition 1251 // again if needed. 1252 if (mAvailability == STATE_KEYPHRASE_ENROLLED) { 1253 try { 1254 stopRecognitionLocked(); 1255 } catch (SecurityException e) { 1256 Slog.w(TAG, "Failed to Stop the recognition", e); 1257 if (mTargetSdkVersion <= Build.VERSION_CODES.R) { 1258 throw e; 1259 } 1260 updateAndNotifyStateChangedLocked(STATE_ERROR); 1261 return; 1262 } 1263 } 1264 1265 // Execute a refresh availability task - which should then notify of a change. 1266 new RefreshAvailabiltyTask().execute(); 1267 } 1268 } 1269 startRecognitionLocked(int recognitionFlags)1270 private int startRecognitionLocked(int recognitionFlags) { 1271 KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; 1272 // TODO: Do we need to do something about the confidence level here? 1273 recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.getId(), 1274 mKeyphraseMetadata.getRecognitionModeFlags(), 0, new ConfidenceLevel[0]); 1275 boolean captureTriggerAudio = 1276 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 1277 boolean allowMultipleTriggers = 1278 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 1279 boolean runInBatterySaver = (recognitionFlags&RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER) != 0; 1280 1281 int audioCapabilities = 0; 1282 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION) != 0) { 1283 audioCapabilities |= AUDIO_CAPABILITY_ECHO_CANCELLATION; 1284 } 1285 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION) != 0) { 1286 audioCapabilities |= AUDIO_CAPABILITY_NOISE_SUPPRESSION; 1287 } 1288 1289 int code; 1290 try { 1291 code = mSoundTriggerSession.startRecognition( 1292 mKeyphraseMetadata.getId(), mLocale.toLanguageTag(), mInternalCallback, 1293 new RecognitionConfig(captureTriggerAudio, allowMultipleTriggers, 1294 recognitionExtra, null /* additional data */, audioCapabilities), 1295 runInBatterySaver); 1296 } catch (RemoteException e) { 1297 throw e.rethrowFromSystemServer(); 1298 } 1299 1300 if (code != STATUS_OK) { 1301 Slog.w(TAG, "startRecognition() failed with error code " + code); 1302 } 1303 return code; 1304 } 1305 stopRecognitionLocked()1306 private int stopRecognitionLocked() { 1307 int code; 1308 try { 1309 code = mSoundTriggerSession.stopRecognition(mKeyphraseMetadata.getId(), 1310 mInternalCallback); 1311 } catch (RemoteException e) { 1312 throw e.rethrowFromSystemServer(); 1313 } 1314 1315 if (code != STATUS_OK) { 1316 Slog.w(TAG, "stopRecognition() failed with error code " + code); 1317 } 1318 return code; 1319 } 1320 setParameterLocked(@odelParams int modelParam, int value)1321 private int setParameterLocked(@ModelParams int modelParam, int value) { 1322 try { 1323 int code = mSoundTriggerSession.setParameter(mKeyphraseMetadata.getId(), modelParam, 1324 value); 1325 1326 if (code != STATUS_OK) { 1327 Slog.w(TAG, "setParameter failed with error code " + code); 1328 } 1329 1330 return code; 1331 } catch (RemoteException e) { 1332 throw e.rethrowFromSystemServer(); 1333 } 1334 } 1335 getParameterLocked(@odelParams int modelParam)1336 private int getParameterLocked(@ModelParams int modelParam) { 1337 try { 1338 return mSoundTriggerSession.getParameter(mKeyphraseMetadata.getId(), modelParam); 1339 } catch (RemoteException e) { 1340 throw e.rethrowFromSystemServer(); 1341 } 1342 } 1343 1344 @Nullable queryParameterLocked(@odelParams int modelParam)1345 private ModelParamRange queryParameterLocked(@ModelParams int modelParam) { 1346 try { 1347 SoundTrigger.ModelParamRange modelParamRange = 1348 mSoundTriggerSession.queryParameter(mKeyphraseMetadata.getId(), modelParam); 1349 1350 if (modelParamRange == null) { 1351 return null; 1352 } 1353 1354 return new ModelParamRange(modelParamRange); 1355 } catch (RemoteException e) { 1356 throw e.rethrowFromSystemServer(); 1357 } 1358 } 1359 updateAndNotifyStateChangedLocked(int availability)1360 private void updateAndNotifyStateChangedLocked(int availability) { 1361 if (DBG) { 1362 Slog.d(TAG, "Hotword availability changed from " + mAvailability 1363 + " -> " + availability); 1364 } 1365 mAvailability = availability; 1366 notifyStateChangedLocked(); 1367 } 1368 notifyStateChangedLocked()1369 private void notifyStateChangedLocked() { 1370 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 1371 message.arg1 = mAvailability; 1372 message.sendToTarget(); 1373 } 1374 1375 /** @hide */ 1376 static final class SoundTriggerListener extends IHotwordRecognitionStatusCallback.Stub { 1377 private final Handler mHandler; 1378 SoundTriggerListener(Handler handler)1379 public SoundTriggerListener(Handler handler) { 1380 mHandler = handler; 1381 } 1382 1383 @Override onKeyphraseDetected( KeyphraseRecognitionEvent event, HotwordDetectedResult result)1384 public void onKeyphraseDetected( 1385 KeyphraseRecognitionEvent event, HotwordDetectedResult result) { 1386 if (DBG) { 1387 Slog.d(TAG, "onDetected(" + event + ")"); 1388 } else { 1389 Slog.i(TAG, "onDetected"); 1390 } 1391 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 1392 new EventPayload.Builder(event) 1393 .setHotwordDetectedResult(result) 1394 .build()) 1395 .sendToTarget(); 1396 } 1397 @Override onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)1398 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 1399 Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event); 1400 } 1401 1402 @Override onRejected(@onNull HotwordRejectedResult result)1403 public void onRejected(@NonNull HotwordRejectedResult result) { 1404 if (DBG) { 1405 Slog.d(TAG, "onRejected(" + result + ")"); 1406 } else { 1407 Slog.i(TAG, "onRejected"); 1408 } 1409 Message.obtain(mHandler, MSG_HOTWORD_REJECTED, result).sendToTarget(); 1410 } 1411 1412 @Override onError(int status)1413 public void onError(int status) { 1414 Slog.i(TAG, "onError: " + status); 1415 mHandler.sendEmptyMessage(MSG_DETECTION_ERROR); 1416 } 1417 1418 @Override onRecognitionPaused()1419 public void onRecognitionPaused() { 1420 Slog.i(TAG, "onRecognitionPaused"); 1421 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 1422 } 1423 1424 @Override onRecognitionResumed()1425 public void onRecognitionResumed() { 1426 Slog.i(TAG, "onRecognitionResumed"); 1427 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 1428 } 1429 1430 @Override onStatusReported(int status)1431 public void onStatusReported(int status) { 1432 if (DBG) { 1433 Slog.d(TAG, "onStatusReported(" + status + ")"); 1434 } else { 1435 Slog.i(TAG, "onStatusReported"); 1436 } 1437 Message message = Message.obtain(mHandler, MSG_HOTWORD_STATUS_REPORTED); 1438 message.arg1 = status; 1439 message.sendToTarget(); 1440 } 1441 1442 @Override onProcessRestarted()1443 public void onProcessRestarted() { 1444 Slog.i(TAG, "onProcessRestarted"); 1445 mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED); 1446 } 1447 } 1448 1449 class MyHandler extends Handler { 1450 @Override handleMessage(Message msg)1451 public void handleMessage(Message msg) { 1452 synchronized (mLock) { 1453 if (mAvailability == STATE_INVALID) { 1454 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 1455 return; 1456 } 1457 } 1458 1459 switch (msg.what) { 1460 case MSG_AVAILABILITY_CHANGED: 1461 mExternalCallback.onAvailabilityChanged(msg.arg1); 1462 break; 1463 case MSG_HOTWORD_DETECTED: 1464 mExternalCallback.onDetected((EventPayload) msg.obj); 1465 break; 1466 case MSG_DETECTION_ERROR: 1467 mExternalCallback.onError(); 1468 break; 1469 case MSG_DETECTION_PAUSE: 1470 mExternalCallback.onRecognitionPaused(); 1471 break; 1472 case MSG_DETECTION_RESUME: 1473 mExternalCallback.onRecognitionResumed(); 1474 break; 1475 case MSG_HOTWORD_REJECTED: 1476 mExternalCallback.onRejected((HotwordRejectedResult) msg.obj); 1477 break; 1478 case MSG_HOTWORD_STATUS_REPORTED: 1479 mExternalCallback.onHotwordDetectionServiceInitialized(msg.arg1); 1480 break; 1481 case MSG_PROCESS_RESTARTED: 1482 mExternalCallback.onHotwordDetectionServiceRestarted(); 1483 break; 1484 default: 1485 super.handleMessage(msg); 1486 } 1487 } 1488 } 1489 1490 class RefreshAvailabiltyTask extends AsyncTask<Void, Void, Void> { 1491 1492 @Override doInBackground(Void... params)1493 public Void doInBackground(Void... params) { 1494 try { 1495 int availability = internalGetInitialAvailability(); 1496 1497 synchronized (mLock) { 1498 if (availability == STATE_NOT_READY) { 1499 internalUpdateEnrolledKeyphraseMetadata(); 1500 if (mKeyphraseMetadata != null) { 1501 availability = STATE_KEYPHRASE_ENROLLED; 1502 } else { 1503 availability = STATE_KEYPHRASE_UNENROLLED; 1504 } 1505 } 1506 updateAndNotifyStateChangedLocked(availability); 1507 } 1508 } catch (SecurityException e) { 1509 Slog.w(TAG, "Failed to refresh availability", e); 1510 if (mTargetSdkVersion <= Build.VERSION_CODES.R) { 1511 throw e; 1512 } 1513 synchronized (mLock) { 1514 updateAndNotifyStateChangedLocked(STATE_ERROR); 1515 } 1516 } 1517 1518 return null; 1519 } 1520 1521 /** 1522 * @return The initial availability without checking the enrollment status. 1523 */ internalGetInitialAvailability()1524 private int internalGetInitialAvailability() { 1525 synchronized (mLock) { 1526 // This detector has already been invalidated. 1527 if (mAvailability == STATE_INVALID) { 1528 return STATE_INVALID; 1529 } 1530 } 1531 1532 ModuleProperties dspModuleProperties; 1533 try { 1534 dspModuleProperties = 1535 mSoundTriggerSession.getDspModuleProperties(); 1536 } catch (RemoteException e) { 1537 throw e.rethrowFromSystemServer(); 1538 } 1539 1540 // No DSP available 1541 if (dspModuleProperties == null) { 1542 return STATE_HARDWARE_UNAVAILABLE; 1543 } 1544 1545 return STATE_NOT_READY; 1546 } 1547 internalUpdateEnrolledKeyphraseMetadata()1548 private void internalUpdateEnrolledKeyphraseMetadata() { 1549 try { 1550 mKeyphraseMetadata = mModelManagementService.getEnrolledKeyphraseMetadata( 1551 mText, mLocale.toLanguageTag()); 1552 } catch (RemoteException e) { 1553 throw e.rethrowFromSystemServer(); 1554 } 1555 } 1556 } 1557 1558 /** @hide */ dump(String prefix, PrintWriter pw)1559 public void dump(String prefix, PrintWriter pw) { 1560 synchronized (mLock) { 1561 pw.print(prefix); pw.print("Text="); pw.println(mText); 1562 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 1563 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 1564 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 1565 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 1566 } 1567 } 1568 } 1569