1 /** 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.service.voice; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 import static android.service.voice.SoundTriggerFailure.ERROR_CODE_UNKNOWN; 22 import static android.service.voice.VoiceInteractionService.MULTIPLE_ACTIVE_HOTWORD_DETECTORS; 23 24 import android.annotation.ElapsedRealtimeLong; 25 import android.annotation.FlaggedApi; 26 import android.annotation.IntDef; 27 import android.annotation.NonNull; 28 import android.annotation.Nullable; 29 import android.annotation.RequiresPermission; 30 import android.annotation.SuppressLint; 31 import android.annotation.SystemApi; 32 import android.annotation.TestApi; 33 import android.app.ActivityThread; 34 import android.app.compat.CompatChanges; 35 import android.compat.annotation.ChangeId; 36 import android.compat.annotation.EnabledSince; 37 import android.compat.annotation.UnsupportedAppUsage; 38 import android.content.Context; 39 import android.content.Intent; 40 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; 41 import android.hardware.soundtrigger.KeyphraseMetadata; 42 import android.hardware.soundtrigger.SoundTrigger; 43 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; 44 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; 45 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 46 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 47 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 48 import android.media.AudioFormat; 49 import android.media.permission.Identity; 50 import android.os.AsyncTask; 51 import android.os.Binder; 52 import android.os.Build; 53 import android.os.Handler; 54 import android.os.HandlerExecutor; 55 import android.os.IBinder; 56 import android.os.Looper; 57 import android.os.Message; 58 import android.os.ParcelFileDescriptor; 59 import android.os.PersistableBundle; 60 import android.os.RemoteException; 61 import android.os.SharedMemory; 62 import android.os.SystemClock; 63 import android.text.TextUtils; 64 import android.util.Log; 65 import android.util.Slog; 66 67 import com.android.internal.annotations.GuardedBy; 68 import com.android.internal.app.IHotwordRecognitionStatusCallback; 69 import com.android.internal.app.IVoiceInteractionManagerService; 70 import com.android.internal.app.IVoiceInteractionSoundTriggerSession; 71 import com.android.internal.infra.AndroidFuture; 72 73 import java.io.PrintWriter; 74 import java.lang.annotation.Retention; 75 import java.lang.annotation.RetentionPolicy; 76 import java.util.ArrayList; 77 import java.util.Arrays; 78 import java.util.Collections; 79 import java.util.HashSet; 80 import java.util.List; 81 import java.util.Locale; 82 import java.util.Objects; 83 import java.util.Set; 84 import java.util.concurrent.Executor; 85 86 /** 87 * A class that lets a VoiceInteractionService implementation interact with 88 * always-on keyphrase detection APIs. 89 * 90 * @hide 91 * TODO(b/168605867): Once Metalava supports expressing a removed public, but current system API, 92 * mark and track it as such. 93 */ 94 @SystemApi 95 public class AlwaysOnHotwordDetector extends AbstractDetector { 96 //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----// 97 /** 98 * Indicates that this hotword detector is no longer valid for any recognition 99 * and should not be used anymore. 100 */ 101 private static final int STATE_INVALID = -3; 102 103 /** 104 * Indicates that recognition for the given keyphrase is not available on the system 105 * because of the hardware configuration. 106 * No further interaction should be performed with the detector that returns this availability. 107 */ 108 public static final int STATE_HARDWARE_UNAVAILABLE = -2; 109 110 /** 111 * Indicates that recognition for the given keyphrase is not supported. 112 * No further interaction should be performed with the detector that returns this availability. 113 * 114 * @deprecated This is no longer a valid state. Enrollment can occur outside of 115 * {@link KeyphraseEnrollmentInfo} through another privileged application. We can no longer 116 * determine ahead of time if the keyphrase and locale are unsupported by the system. 117 */ 118 @Deprecated 119 public static final int STATE_KEYPHRASE_UNSUPPORTED = -1; 120 121 /** 122 * Indicates that the given keyphrase is not enrolled. 123 * The caller may choose to begin an enrollment flow for the keyphrase. 124 */ 125 public static final int STATE_KEYPHRASE_UNENROLLED = 1; 126 127 /** 128 * Indicates that the given keyphrase is currently enrolled and it's possible to start 129 * recognition for it. 130 */ 131 public static final int STATE_KEYPHRASE_ENROLLED = 2; 132 133 /** 134 * Indicates that the availability state of the active keyphrase can't be known due to an error. 135 * 136 * <p>NOTE: No further interaction should be performed with the detector that returns this 137 * state, it would be better to create {@link AlwaysOnHotwordDetector} again. 138 */ 139 public static final int STATE_ERROR = 3; 140 141 /** 142 * Indicates that the detector isn't ready currently. 143 */ 144 private static final int STATE_NOT_READY = 0; 145 146 //-- Flags for startRecognition ----// 147 /** @hide */ 148 @Retention(RetentionPolicy.SOURCE) 149 @IntDef(flag = true, prefix = { "RECOGNITION_FLAG_" }, value = { 150 RECOGNITION_FLAG_NONE, 151 RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO, 152 RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS, 153 RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION, 154 RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION, 155 RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER, 156 }) 157 public @interface RecognitionFlags {} 158 159 /** 160 * Empty flag for {@link #startRecognition(int)}. 161 * 162 * @hide 163 */ 164 public static final int RECOGNITION_FLAG_NONE = 0; 165 166 /** 167 * Recognition flag for {@link #startRecognition(int)} that indicates 168 * whether the trigger audio for hotword needs to be captured. 169 */ 170 public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; 171 172 /** 173 * Recognition flag for {@link #startRecognition(int)} that indicates 174 * whether the recognition should keep going on even after the keyphrase triggers. 175 * If this flag is specified, it's possible to get multiple triggers after a 176 * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times. 177 * When this isn't specified, the default behavior is to stop recognition once the 178 * keyphrase is spoken, till the caller starts recognition again. 179 */ 180 public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2; 181 182 /** 183 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 184 * if the underlying recognition should use AEC. 185 * This capability may or may not be supported by the system, and support can be queried 186 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 187 * this flag is {@link #AUDIO_CAPABILITY_ECHO_CANCELLATION}. If this flag is passed without the 188 * audio capability supported, there will be no audio effect applied. 189 */ 190 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION = 0x4; 191 192 /** 193 * Audio capabilities flag for {@link #startRecognition(int)} that indicates 194 * if the underlying recognition should use noise suppression. 195 * This capability may or may not be supported by the system, and support can be queried 196 * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for 197 * this flag is {@link #AUDIO_CAPABILITY_NOISE_SUPPRESSION}. If this flag is passed without the 198 * audio capability supported, there will be no audio effect applied. 199 */ 200 public static final int RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION = 0x8; 201 202 /** 203 * Recognition flag for {@link #startRecognition(int)} that indicates whether the recognition 204 * should continue after battery saver mode is enabled. 205 * When this flag is specified, the caller will be checked for 206 * {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER} permission granted. 207 */ 208 public static final int RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER = 0x10; 209 210 //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----// 211 // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. 212 213 /** @hide */ 214 @Retention(RetentionPolicy.SOURCE) 215 @IntDef(flag = true, prefix = { "RECOGNITION_MODE_" }, value = { 216 RECOGNITION_MODE_VOICE_TRIGGER, 217 RECOGNITION_MODE_USER_IDENTIFICATION, 218 }) 219 public @interface RecognitionModes {} 220 221 /** 222 * Simple recognition of the key phrase. 223 * Returned by {@link #getSupportedRecognitionModes()} 224 */ 225 public static final int RECOGNITION_MODE_VOICE_TRIGGER 226 = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; 227 228 /** 229 * User identification performed with the keyphrase recognition. 230 * Returned by {@link #getSupportedRecognitionModes()} 231 */ 232 public static final int RECOGNITION_MODE_USER_IDENTIFICATION 233 = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; 234 235 //-- Audio capabilities. Values in returned bit field for getSupportedAudioCapabilities() --// 236 237 /** @hide */ 238 @Retention(RetentionPolicy.SOURCE) 239 @IntDef(flag = true, prefix = { "AUDIO_CAPABILITY_" }, value = { 240 AUDIO_CAPABILITY_ECHO_CANCELLATION, 241 AUDIO_CAPABILITY_NOISE_SUPPRESSION, 242 }) 243 public @interface AudioCapabilities {} 244 245 /** 246 * If set the underlying module supports AEC. 247 * Returned by {@link #getSupportedAudioCapabilities()} 248 */ 249 public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION = 250 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_ECHO_CANCELLATION; 251 252 /** 253 * If set, the underlying module supports noise suppression. 254 * Returned by {@link #getSupportedAudioCapabilities()} 255 */ 256 public static final int AUDIO_CAPABILITY_NOISE_SUPPRESSION = 257 SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_NOISE_SUPPRESSION; 258 259 /** @hide */ 260 @Retention(RetentionPolicy.SOURCE) 261 @IntDef(flag = true, prefix = { "MODEL_PARAM_" }, value = { 262 MODEL_PARAM_THRESHOLD_FACTOR, 263 }) 264 public @interface ModelParams {} 265 266 /** 267 * Gates returning {@code IllegalStateException} in {@link #initialize( 268 * PersistableBundle, SharedMemory, SoundTrigger.ModuleProperties)} when no DSP module 269 * is available. If the change is not enabled, the existing behavior of not throwing an 270 * exception and delivering {@link STATE_HARDWARE_UNAVAILABLE} is retained. 271 */ 272 @ChangeId 273 @EnabledSince(targetSdkVersion = Build.VERSION_CODES.UPSIDE_DOWN_CAKE) 274 static final long THROW_ON_INITIALIZE_IF_NO_DSP = 269165460L; 275 276 /** 277 * Gates returning {@link Callback#onFailure} and {@link Callback#onUnknownFailure} 278 * when asynchronous exceptions are propagated to the client. If the change is not enabled, 279 * the existing behavior of delivering {@link #STATE_ERROR} is retained. 280 */ 281 @ChangeId 282 @EnabledSince(targetSdkVersion = Build.VERSION_CODES.UPSIDE_DOWN_CAKE) 283 static final long SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS = 280471513L; 284 285 /** 286 * Controls the sensitivity threshold adjustment factor for a given model. 287 * Negative value corresponds to less sensitive model (high threshold) and 288 * a positive value corresponds to a more sensitive model (low threshold). 289 * Default value is 0. 290 */ 291 public static final int MODEL_PARAM_THRESHOLD_FACTOR = 292 android.hardware.soundtrigger.ModelParams.THRESHOLD_FACTOR; 293 294 static final String TAG = "AlwaysOnHotwordDetector"; 295 static final boolean DBG = false; 296 297 private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; 298 private static final int STATUS_OK = SoundTrigger.STATUS_OK; 299 300 private static final int MSG_AVAILABILITY_CHANGED = 1; 301 private static final int MSG_HOTWORD_DETECTED = 2; 302 private static final int MSG_DETECTION_ERROR = 3; 303 private static final int MSG_DETECTION_PAUSE = 4; 304 private static final int MSG_DETECTION_RESUME = 5; 305 private static final int MSG_HOTWORD_REJECTED = 6; 306 private static final int MSG_HOTWORD_STATUS_REPORTED = 7; 307 private static final int MSG_PROCESS_RESTARTED = 8; 308 private static final int MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE = 9; 309 private static final int MSG_DETECTION_SOUND_TRIGGER_FAILURE = 10; 310 private static final int MSG_DETECTION_UNKNOWN_FAILURE = 11; 311 312 private final String mText; 313 private final Locale mLocale; 314 /** 315 * The metadata of the Keyphrase, derived from the enrollment application. 316 * This may be null if this keyphrase isn't supported by the enrollment application. 317 */ 318 @GuardedBy("mLock") 319 @Nullable 320 private KeyphraseMetadata mKeyphraseMetadata; 321 private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; 322 private final IVoiceInteractionManagerService mModelManagementService; 323 private IVoiceInteractionSoundTriggerSession mSoundTriggerSession; 324 private final SoundTriggerListener mInternalCallback; 325 private final Callback mExternalCallback; 326 private final Executor mExternalExecutor; 327 private final Handler mHandler; 328 private final IBinder mBinder = new Binder(); 329 private final boolean mSupportSandboxedDetectionService; 330 private final String mAttributionTag; 331 332 @GuardedBy("mLock") 333 private boolean mIsAvailabilityOverriddenByTestApi = false; 334 @GuardedBy("mLock") 335 private int mAvailability = STATE_NOT_READY; 336 337 /** 338 * A ModelParamRange is a representation of supported parameter range for a 339 * given loaded model. 340 */ 341 public static final class ModelParamRange { 342 private final SoundTrigger.ModelParamRange mModelParamRange; 343 344 /** @hide */ ModelParamRange(SoundTrigger.ModelParamRange modelParamRange)345 ModelParamRange(SoundTrigger.ModelParamRange modelParamRange) { 346 mModelParamRange = modelParamRange; 347 } 348 349 /** 350 * Get the beginning of the param range 351 * 352 * @return The inclusive start of the supported range. 353 */ getStart()354 public int getStart() { 355 return mModelParamRange.getStart(); 356 } 357 358 /** 359 * Get the end of the param range 360 * 361 * @return The inclusive end of the supported range. 362 */ getEnd()363 public int getEnd() { 364 return mModelParamRange.getEnd(); 365 } 366 367 @Override 368 @NonNull toString()369 public String toString() { 370 return mModelParamRange.toString(); 371 } 372 373 @Override equals(@ullable Object obj)374 public boolean equals(@Nullable Object obj) { 375 return mModelParamRange.equals(obj); 376 } 377 378 @Override hashCode()379 public int hashCode() { 380 return mModelParamRange.hashCode(); 381 } 382 } 383 384 /** 385 * Additional payload for {@link Callback#onDetected}. 386 */ 387 public static class EventPayload { 388 389 /** 390 * Flags for describing the data format provided in the event payload. 391 * 392 * @hide 393 */ 394 @Retention(RetentionPolicy.SOURCE) 395 @IntDef(prefix = {"DATA_FORMAT_"}, value = { 396 DATA_FORMAT_RAW, 397 DATA_FORMAT_TRIGGER_AUDIO, 398 }) 399 public @interface DataFormat { 400 } 401 402 /** 403 * Data format is not strictly defined by the framework, and the 404 * {@link android.hardware.soundtrigger.SoundTriggerModule} voice engine may populate this 405 * field in any format. 406 */ 407 public static final int DATA_FORMAT_RAW = 0; 408 409 /** 410 * Data format is defined as trigger audio. 411 * 412 * <p>When this format is used, {@link #getCaptureAudioFormat()} can be used to understand 413 * further the audio format for reading the data. 414 * 415 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 416 */ 417 public static final int DATA_FORMAT_TRIGGER_AUDIO = 1; 418 419 @DataFormat 420 private final int mDataFormat; 421 // Indicates if {@code captureSession} can be used to continue capturing more audio 422 // from the DSP hardware. 423 private final boolean mCaptureAvailable; 424 // The session to use when attempting to capture more audio from the DSP hardware. 425 private final int mCaptureSession; 426 private final AudioFormat mAudioFormat; 427 // Raw data associated with the event. 428 // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true. 429 private final byte[] mData; 430 private final HotwordDetectedResult mHotwordDetectedResult; 431 private final ParcelFileDescriptor mAudioStream; 432 private final List<KeyphraseRecognitionExtra> mKephraseExtras; 433 434 @ElapsedRealtimeLong 435 private final long mHalEventReceivedMillis; 436 437 private final boolean mIsRecognitionStopped; 438 EventPayload( boolean captureAvailable, @Nullable AudioFormat audioFormat, int captureSession, @DataFormat int dataFormat, @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, @Nullable ParcelFileDescriptor audioStream, @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras, @ElapsedRealtimeLong long halEventReceivedMillis, boolean isRecognitionStopped)439 private EventPayload( 440 boolean captureAvailable, 441 @Nullable AudioFormat audioFormat, 442 int captureSession, 443 @DataFormat int dataFormat, 444 @Nullable byte[] data, 445 @Nullable HotwordDetectedResult hotwordDetectedResult, 446 @Nullable ParcelFileDescriptor audioStream, 447 @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras, 448 @ElapsedRealtimeLong long halEventReceivedMillis, 449 boolean isRecognitionStopped) { 450 mCaptureAvailable = captureAvailable; 451 mCaptureSession = captureSession; 452 mAudioFormat = audioFormat; 453 mDataFormat = dataFormat; 454 mData = data; 455 mHotwordDetectedResult = hotwordDetectedResult; 456 mAudioStream = audioStream; 457 mKephraseExtras = keyphraseExtras; 458 mHalEventReceivedMillis = halEventReceivedMillis; 459 mIsRecognitionStopped = isRecognitionStopped; 460 } 461 462 /** 463 * Gets the format of the audio obtained using {@link #getTriggerAudio()}. 464 * May be null if there's no audio present. 465 */ 466 @Nullable getCaptureAudioFormat()467 public AudioFormat getCaptureAudioFormat() { 468 return mAudioFormat; 469 } 470 471 /** 472 * Gets the raw audio that triggered the keyphrase. 473 * This may be null if the trigger audio isn't available. 474 * If non-null, the format of the audio can be obtained by calling 475 * {@link #getCaptureAudioFormat()}. 476 * 477 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 478 * @deprecated Use {@link #getData()} instead. 479 */ 480 @Deprecated 481 @Nullable getTriggerAudio()482 public byte[] getTriggerAudio() { 483 if (mDataFormat == DATA_FORMAT_TRIGGER_AUDIO) { 484 return mData; 485 } else { 486 return null; 487 } 488 } 489 490 /** 491 * Conveys the format of the additional data that is triggered with the keyphrase event. 492 * 493 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 494 * @see DataFormat 495 */ 496 @DataFormat getDataFormat()497 public int getDataFormat() { 498 return mDataFormat; 499 } 500 501 /** 502 * Gets additional raw data that is triggered with the keyphrase event. 503 * 504 * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this 505 * field with opaque data for use by system applications who know about voice 506 * engine internals. Data may be null if the field is not populated by the 507 * {@link android.hardware.soundtrigger.SoundTriggerModule}. 508 * 509 * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the 510 * entirety of this buffer is expected to be of the format from 511 * {@link #getCaptureAudioFormat()}. 512 * 513 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 514 */ 515 @Nullable getData()516 public byte[] getData() { 517 return mData; 518 } 519 520 /** 521 * Gets the session ID to start a capture from the DSP. 522 * This may be null if streaming capture isn't possible. 523 * If non-null, the format of the audio that can be captured can be 524 * obtained using {@link #getCaptureAudioFormat()}. 525 * 526 * TODO: Candidate for Public API when the API to start capture with a session ID 527 * is made public. 528 * 529 * TODO: Add this to {@link #getCaptureAudioFormat()}: 530 * "Gets the format of the audio obtained using {@link #getTriggerAudio()} 531 * or {@link #getCaptureSession()}. May be null if no audio can be obtained 532 * for either the trigger or a streaming session." 533 * 534 * TODO: Should this return a known invalid value instead? 535 * 536 * @hide 537 */ 538 @Nullable 539 @UnsupportedAppUsage getCaptureSession()540 public Integer getCaptureSession() { 541 if (mCaptureAvailable) { 542 return mCaptureSession; 543 } else { 544 return null; 545 } 546 } 547 548 /** 549 * Returns {@link HotwordDetectedResult} associated with the hotword event, passed from 550 * {@link HotwordDetectionService}. 551 */ 552 @Nullable getHotwordDetectedResult()553 public HotwordDetectedResult getHotwordDetectedResult() { 554 return mHotwordDetectedResult; 555 } 556 557 /** 558 * Returns a stream with bytes corresponding to the open audio stream with hotword data. 559 * 560 * <p>This data represents an audio stream in the format returned by 561 * {@link #getCaptureAudioFormat}. 562 * 563 * <p>Clients are expected to start consuming the stream within 1 second of receiving the 564 * event. 565 * 566 * <p>When this method returns a non-null, clients must close this stream when it's no 567 * longer needed. Failing to do so will result in microphone being open for longer periods 568 * of time, and app being attributed for microphone usage. 569 */ 570 @Nullable getAudioStream()571 public ParcelFileDescriptor getAudioStream() { 572 return mAudioStream; 573 } 574 575 /** 576 * Returns the keyphrases recognized by the voice engine with additional confidence 577 * information 578 * 579 * @return List of keyphrase extras describing additional data for each keyphrase the voice 580 * engine triggered on for this event. The ordering of the list is preserved based on what 581 * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}. 582 */ 583 @NonNull getKeyphraseRecognitionExtras()584 public List<KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras() { 585 return mKephraseExtras; 586 } 587 588 /** 589 * Timestamp of when the trigger event from SoundTriggerHal was received by the framework. 590 * 591 * Clock monotonic including suspend time or its equivalent on the system, 592 * in the same units and timebase as {@link SystemClock#elapsedRealtime()}. 593 * 594 * @return Elapsed realtime in milliseconds when the event was received from the HAL. 595 * Returns -1 if the event was not generated from the HAL. 596 */ 597 @ElapsedRealtimeLong getHalEventReceivedMillis()598 public long getHalEventReceivedMillis() { 599 return mHalEventReceivedMillis; 600 } 601 602 /** Returns whether the system has stopped hotword recognition because of this detection. */ 603 @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API) isRecognitionStopped()604 public boolean isRecognitionStopped() { 605 return mIsRecognitionStopped; 606 } 607 608 /** 609 * Builder class for {@link EventPayload} objects 610 * 611 * @hide 612 */ 613 @TestApi 614 public static final class Builder { 615 private boolean mCaptureAvailable = false; 616 private int mCaptureSession = -1; 617 private AudioFormat mAudioFormat = null; 618 @DataFormat 619 private int mDataFormat = DATA_FORMAT_RAW; 620 private byte[] mData = null; 621 private HotwordDetectedResult mHotwordDetectedResult = null; 622 private ParcelFileDescriptor mAudioStream = null; 623 private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList(); 624 @ElapsedRealtimeLong 625 private long mHalEventReceivedMillis = -1; 626 // default to true to keep prior behavior 627 private boolean mIsRecognitionStopped = true; 628 Builder()629 public Builder() {} 630 Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent)631 Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) { 632 setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable()); 633 setCaptureSession(keyphraseRecognitionEvent.getCaptureSession()); 634 if (keyphraseRecognitionEvent.getCaptureFormat() != null) { 635 setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat()); 636 } 637 setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO 638 : DATA_FORMAT_RAW); 639 if (keyphraseRecognitionEvent.getData() != null) { 640 setData(keyphraseRecognitionEvent.getData()); 641 } 642 if (keyphraseRecognitionEvent.keyphraseExtras != null) { 643 setKeyphraseRecognitionExtras( 644 Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras)); 645 } 646 setHalEventReceivedMillis(keyphraseRecognitionEvent.getHalEventReceivedMillis()); 647 } 648 649 /** 650 * Indicates if {@code captureSession} can be used to continue capturing more audio from 651 * the DSP hardware. 652 */ 653 @SuppressLint("MissingGetterMatchingBuilder") 654 @NonNull setCaptureAvailable(boolean captureAvailable)655 public Builder setCaptureAvailable(boolean captureAvailable) { 656 mCaptureAvailable = captureAvailable; 657 return this; 658 } 659 660 /** 661 * Sets the session ID to start a capture from the DSP. 662 */ 663 @SuppressLint("MissingGetterMatchingBuilder") 664 @NonNull setCaptureSession(int captureSession)665 public Builder setCaptureSession(int captureSession) { 666 mCaptureSession = captureSession; 667 return this; 668 } 669 670 /** 671 * Sets the format of the audio obtained using {@link #getTriggerAudio()}. 672 */ 673 @NonNull setCaptureAudioFormat(@onNull AudioFormat audioFormat)674 public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) { 675 mAudioFormat = audioFormat; 676 return this; 677 } 678 679 /** 680 * Conveys the format of the additional data that is triggered with the keyphrase event. 681 * 682 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 683 * @see DataFormat 684 */ 685 @NonNull setDataFormat(@ataFormat int dataFormat)686 public Builder setDataFormat(@DataFormat int dataFormat) { 687 mDataFormat = dataFormat; 688 return this; 689 } 690 691 /** 692 * Sets additional raw data that is triggered with the keyphrase event. 693 * 694 * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this 695 * field with opaque data for use by system applications who know about voice 696 * engine internals. Data may be null if the field is not populated by the 697 * {@link android.hardware.soundtrigger.SoundTriggerModule}. 698 * 699 * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the 700 * entirety of this 701 * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}. 702 * 703 * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 704 */ 705 @NonNull setData(@onNull byte[] data)706 public Builder setData(@NonNull byte[] data) { 707 mData = data; 708 return this; 709 } 710 711 /** 712 * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from 713 * {@link HotwordDetectionService}. 714 */ 715 @NonNull setHotwordDetectedResult( @onNull HotwordDetectedResult hotwordDetectedResult)716 public Builder setHotwordDetectedResult( 717 @NonNull HotwordDetectedResult hotwordDetectedResult) { 718 mHotwordDetectedResult = hotwordDetectedResult; 719 return this; 720 } 721 722 /** 723 * Sets a stream with bytes corresponding to the open audio stream with hotword data. 724 * 725 * <p>This data represents an audio stream in the format returned by 726 * {@link #getCaptureAudioFormat}. 727 * 728 * <p>Clients are expected to start consuming the stream within 1 second of receiving 729 * the 730 * event. 731 */ 732 @NonNull setAudioStream(@onNull ParcelFileDescriptor audioStream)733 public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) { 734 mAudioStream = audioStream; 735 return this; 736 } 737 738 /** 739 * Sets the keyphrases recognized by the voice engine with additional confidence 740 * information 741 */ 742 @NonNull setKeyphraseRecognitionExtras( @onNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)743 public Builder setKeyphraseRecognitionExtras( 744 @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) { 745 mKeyphraseExtras = keyphraseRecognitionExtras; 746 return this; 747 } 748 749 /** 750 * Timestamp of when the trigger event from SoundTriggerHal was received by the 751 * framework. 752 * 753 * Clock monotonic including suspend time or its equivalent on the system, 754 * in the same units and timebase as {@link SystemClock#elapsedRealtime()}. 755 */ 756 @NonNull setHalEventReceivedMillis( @lapsedRealtimeLong long halEventReceivedMillis)757 public Builder setHalEventReceivedMillis( 758 @ElapsedRealtimeLong long halEventReceivedMillis) { 759 mHalEventReceivedMillis = halEventReceivedMillis; 760 return this; 761 } 762 763 /** 764 * Sets whether the system has stopped hotword recognition because of this detection. 765 */ 766 @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API) 767 @NonNull setIsRecognitionStopped(boolean isRecognitionStopped)768 public Builder setIsRecognitionStopped(boolean isRecognitionStopped) { 769 mIsRecognitionStopped = isRecognitionStopped; 770 return this; 771 } 772 773 /** 774 * Builds an {@link EventPayload} instance 775 */ 776 @NonNull build()777 public EventPayload build() { 778 return new EventPayload( 779 mCaptureAvailable, 780 mAudioFormat, 781 mCaptureSession, 782 mDataFormat, 783 mData, 784 mHotwordDetectedResult, 785 mAudioStream, 786 mKeyphraseExtras, 787 mHalEventReceivedMillis, 788 mIsRecognitionStopped); 789 } 790 } 791 } 792 793 /** 794 * Callbacks for always-on hotword detection. 795 */ 796 public abstract static class Callback implements HotwordDetector.Callback { 797 798 /** 799 * Updates the availability state of the active keyphrase and locale on every keyphrase 800 * sound model change. 801 * 802 * <p>This API is called whenever there's a possibility that the keyphrase associated 803 * with this detector has been updated. It is not guaranteed that there is in fact any 804 * change, as it may be called for other reasons.</p> 805 * 806 * <p>This API is also guaranteed to be called right after an AlwaysOnHotwordDetector 807 * instance is created to updated the current availability state.</p> 808 * 809 * <p>Availability implies the current enrollment state of the given keyphrase. If the 810 * hardware on this system is not capable of listening for the given keyphrase, 811 * {@link AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE} will be returned. 812 * 813 * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE 814 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED 815 * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED 816 * @see AlwaysOnHotwordDetector#STATE_ERROR 817 */ onAvailabilityChanged(int status)818 public abstract void onAvailabilityChanged(int status); 819 820 /** 821 * Called when the keyphrase is spoken. 822 * 823 * <p>If {@code eventPayload.isRecognitionStopped()} returns true, this implicitly stops 824 * listening for the keyphrase once it's detected. Clients should start a recognition again 825 * once they are done handling this detection. 826 * 827 * @param eventPayload Payload data for the detection event. This may contain the trigger 828 * audio, if requested when calling {@link 829 * AlwaysOnHotwordDetector#startRecognition(int)} or if the audio comes from the {@link 830 * android.service.wearable.WearableSensingService}. 831 */ onDetected(@onNull EventPayload eventPayload)832 public abstract void onDetected(@NonNull EventPayload eventPayload); 833 834 /** 835 * {@inheritDoc} 836 * 837 * @deprecated On {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE} and above, 838 * implement {@link HotwordDetector.Callback#onFailure(HotwordDetectionServiceFailure)}, 839 * {@link AlwaysOnHotwordDetector.Callback#onFailure(SoundTriggerFailure)}, 840 * {@link HotwordDetector.Callback#onUnknownFailure(String)} instead. 841 */ 842 @Deprecated 843 @Override onError()844 public abstract void onError(); 845 846 /** 847 * Called when the detection fails due to an error occurs in the 848 * {@link com.android.server.soundtrigger.SoundTriggerService} and 849 * {@link com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareService}, 850 * {@link SoundTriggerFailure} will be reported to the detector. 851 * 852 * @param soundTriggerFailure It provides the error code, error message and suggested 853 * action. 854 */ onFailure(@onNull SoundTriggerFailure soundTriggerFailure)855 public void onFailure(@NonNull SoundTriggerFailure soundTriggerFailure) { 856 onError(); 857 } 858 859 /** {@inheritDoc} */ onRecognitionPaused()860 public abstract void onRecognitionPaused(); 861 862 /** {@inheritDoc} */ onRecognitionResumed()863 public abstract void onRecognitionResumed(); 864 865 /** {@inheritDoc} */ onRejected(@onNull HotwordRejectedResult result)866 public void onRejected(@NonNull HotwordRejectedResult result) { 867 } 868 869 /** {@inheritDoc} */ onHotwordDetectionServiceInitialized(int status)870 public void onHotwordDetectionServiceInitialized(int status) { 871 } 872 873 /** {@inheritDoc} */ onHotwordDetectionServiceRestarted()874 public void onHotwordDetectionServiceRestarted() { 875 } 876 } 877 878 /** 879 * @param text The keyphrase text to get the detector for. 880 * @param locale The java locale for the detector. 881 * @param callback A non-null Callback for receiving the recognition events. 882 * @param modelManagementService A service that allows management of sound models. 883 * @param targetSdkVersion The target SDK version. 884 * @param SupportSandboxedDetectionService {@code true} if HotwordDetectionService should be 885 * triggered, otherwise {@code false}. 886 * @param attributionTag an optional attribution tag passed form the 887 * {@link VoiceInteractionService} context via the 888 * {@link createAlwaysOnHotwordDetectorInternal(String, Locale, boolean, PersistableBundle, 889 * SharedMemory, ModuleProperties, Executor, Callback)}. 890 * 891 * @hide 892 */ AlwaysOnHotwordDetector(String text, Locale locale, Executor executor, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, boolean supportSandboxedDetectionService, @Nullable String attributionTag)893 public AlwaysOnHotwordDetector(String text, Locale locale, Executor executor, Callback callback, 894 KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, 895 IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, 896 boolean supportSandboxedDetectionService, @Nullable String attributionTag) { 897 super(modelManagementService, executor, callback); 898 899 mHandler = new MyHandler(Looper.getMainLooper()); 900 mText = text; 901 mLocale = locale; 902 mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; 903 mExternalCallback = callback; 904 mExternalExecutor = executor != null ? executor : new HandlerExecutor( 905 new Handler(Looper.myLooper())); 906 mInternalCallback = new SoundTriggerListener(mHandler); 907 mModelManagementService = modelManagementService; 908 mSupportSandboxedDetectionService = supportSandboxedDetectionService; 909 mAttributionTag = attributionTag; 910 } 911 912 // Do nothing. This method should not be abstract. 913 // TODO (b/269355519) un-subclass AOHD. 914 @Override initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)915 void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {} 916 initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @Nullable SoundTrigger.ModuleProperties moduleProperties)917 void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory, 918 @Nullable SoundTrigger.ModuleProperties moduleProperties) { 919 if (mSupportSandboxedDetectionService) { 920 initAndVerifyDetector(options, sharedMemory, mInternalCallback, 921 DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, mAttributionTag); 922 } 923 try { 924 Identity identity = new Identity(); 925 identity.packageName = ActivityThread.currentOpPackageName(); 926 if (IS_IDENTITY_WITH_ATTRIBUTION_TAG) { 927 identity.attributionTag = mAttributionTag; 928 } 929 if (moduleProperties == null) { 930 moduleProperties = mModelManagementService 931 .listModuleProperties(identity) 932 .stream() 933 .filter(prop -> !prop.getSupportedModelArch() 934 .equals(SoundTrigger.FAKE_HAL_ARCH)) 935 .findFirst() 936 .orElse(null); 937 if (CompatChanges.isChangeEnabled(THROW_ON_INITIALIZE_IF_NO_DSP) && 938 moduleProperties == null) { 939 throw new IllegalStateException("No DSP module available to attach to"); 940 } 941 } 942 mSoundTriggerSession = 943 mModelManagementService.createSoundTriggerSessionAsOriginator( 944 identity, mBinder, moduleProperties); 945 } catch (RemoteException e) { 946 throw e.rethrowAsRuntimeException(); 947 } 948 new RefreshAvailabilityTask().execute(); 949 } 950 951 /** 952 * {@inheritDoc} 953 * 954 * @throws IllegalStateException if this AlwaysOnHotwordDetector wasn't specified to use a 955 * {@link HotwordDetectionService} when it was created. In addition, if this 956 * AlwaysOnHotwordDetector is in an invalid or error state. 957 */ 958 @Override updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)959 public final void updateState(@Nullable PersistableBundle options, 960 @Nullable SharedMemory sharedMemory) { 961 synchronized (mLock) { 962 if (!mSupportSandboxedDetectionService) { 963 throw new IllegalStateException( 964 "updateState called, but it doesn't support hotword detection service"); 965 } 966 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 967 throw new IllegalStateException( 968 "updateState called on an invalid detector or error state"); 969 } 970 } 971 972 super.updateState(options, sharedMemory); 973 } 974 975 /** 976 * Test API for manipulating the voice engine and sound model availability. 977 * 978 * After overriding the availability status, the client's 979 * {@link Callback#onAvailabilityChanged(int)} will be called to reflect the updated state. 980 * 981 * When this override is set, all system updates to availability will be ignored. 982 * @hide 983 */ 984 @TestApi overrideAvailability(int availability)985 public void overrideAvailability(int availability) { 986 synchronized (mLock) { 987 mAvailability = availability; 988 mIsAvailabilityOverriddenByTestApi = true; 989 // ENROLLED state requires there to be metadata about the sound model so a fake one 990 // is created. 991 if (mKeyphraseMetadata == null && mAvailability == STATE_KEYPHRASE_ENROLLED) { 992 Set<Locale> fakeSupportedLocales = new HashSet<>(); 993 fakeSupportedLocales.add(mLocale); 994 mKeyphraseMetadata = new KeyphraseMetadata(1, mText, fakeSupportedLocales, 995 AlwaysOnHotwordDetector.RECOGNITION_MODE_VOICE_TRIGGER); 996 } 997 notifyStateChangedLocked(); 998 } 999 } 1000 1001 /** 1002 * Test API for clearing an availability override set by {@link #overrideAvailability(int)} 1003 * 1004 * This method will restore the availability to the current system state. 1005 * @hide 1006 */ 1007 @TestApi resetAvailability()1008 public void resetAvailability() { 1009 synchronized (mLock) { 1010 mIsAvailabilityOverriddenByTestApi = false; 1011 } 1012 // Execute a refresh availability task - which should then notify of a change. 1013 new RefreshAvailabilityTask().execute(); 1014 } 1015 1016 /** 1017 * Test API to simulate to trigger hardware recognition event for test. 1018 * 1019 * @hide 1020 */ 1021 @TestApi 1022 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, @ElapsedRealtimeLong long halEventReceivedMillis, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)1023 public void triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, 1024 @ElapsedRealtimeLong long halEventReceivedMillis, boolean captureAvailable, 1025 int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, 1026 @NonNull AudioFormat captureFormat, @Nullable byte[] data, 1027 @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) { 1028 Log.d(TAG, "triggerHardwareRecognitionEventForTest()"); 1029 synchronized (mLock) { 1030 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1031 throw new IllegalStateException("triggerHardwareRecognitionEventForTest called on" 1032 + " an invalid detector or error state"); 1033 } 1034 try { 1035 mModelManagementService.triggerHardwareRecognitionEventForTest( 1036 new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable, 1037 captureSession, captureDelayMs, capturePreambleMs, triggerInData, 1038 captureFormat, data, keyphraseRecognitionExtras.toArray( 1039 new KeyphraseRecognitionExtra[0]), halEventReceivedMillis, 1040 new Binder()), 1041 mInternalCallback); 1042 } catch (RemoteException e) { 1043 throw e.rethrowFromSystemServer(); 1044 } 1045 } 1046 } 1047 1048 /** 1049 * Gets the recognition modes supported by the associated keyphrase. 1050 * 1051 * @see #RECOGNITION_MODE_USER_IDENTIFICATION 1052 * @see #RECOGNITION_MODE_VOICE_TRIGGER 1053 * 1054 * @throws UnsupportedOperationException if the keyphrase itself isn't supported. 1055 * Callers should only call this method after a supported state callback on 1056 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1057 * @throws IllegalStateException if the detector is in an invalid or error state. 1058 * This may happen if another detector has been instantiated or the 1059 * {@link VoiceInteractionService} hosting this detector has been shut down. 1060 */ getSupportedRecognitionModes()1061 public @RecognitionModes int getSupportedRecognitionModes() { 1062 if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()"); 1063 synchronized (mLock) { 1064 return getSupportedRecognitionModesLocked(); 1065 } 1066 } 1067 1068 @GuardedBy("mLock") getSupportedRecognitionModesLocked()1069 private int getSupportedRecognitionModesLocked() { 1070 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1071 throw new IllegalStateException( 1072 "getSupportedRecognitionModes called on an invalid detector or error state"); 1073 } 1074 1075 // This method only makes sense if we can actually support a recognition. 1076 if (mAvailability != STATE_KEYPHRASE_ENROLLED || mKeyphraseMetadata == null) { 1077 throw new UnsupportedOperationException( 1078 "Getting supported recognition modes for the keyphrase is not supported"); 1079 } 1080 1081 return mKeyphraseMetadata.getRecognitionModeFlags(); 1082 } 1083 1084 /** 1085 * Get the audio capabilities supported by the platform which can be enabled when 1086 * starting a recognition. 1087 * Caller must be the active voice interaction service via 1088 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1089 * 1090 * @see #AUDIO_CAPABILITY_ECHO_CANCELLATION 1091 * @see #AUDIO_CAPABILITY_NOISE_SUPPRESSION 1092 * 1093 * @return Bit field encoding of the AudioCapabilities supported. 1094 */ 1095 @AudioCapabilities getSupportedAudioCapabilities()1096 public int getSupportedAudioCapabilities() { 1097 if (DBG) Slog.d(TAG, "getSupportedAudioCapabilities()"); 1098 synchronized (mLock) { 1099 return getSupportedAudioCapabilitiesLocked(); 1100 } 1101 } 1102 1103 @GuardedBy("mLock") getSupportedAudioCapabilitiesLocked()1104 private int getSupportedAudioCapabilitiesLocked() { 1105 try { 1106 ModuleProperties properties = 1107 mSoundTriggerSession.getDspModuleProperties(); 1108 if (properties != null) { 1109 return properties.getAudioCapabilities(); 1110 } 1111 1112 return 0; 1113 } catch (RemoteException e) { 1114 throw e.rethrowFromSystemServer(); 1115 } 1116 } 1117 1118 /** 1119 * Starts recognition for the associated keyphrase. 1120 * Caller must be the active voice interaction service via 1121 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1122 * 1123 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 1124 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 1125 * 1126 * @param recognitionFlags The flags to control the recognition properties. 1127 * @param data Additional pass-through data to the system voice engine along with the 1128 * startRecognition request. This data is intended to provide additional parameters 1129 * when starting the opaque sound model. 1130 * @return Indicates whether the call succeeded or not. 1131 * @throws UnsupportedOperationException if the recognition isn't supported. 1132 * Callers should only call this method after a supported state callback on 1133 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1134 * @throws IllegalStateException if the detector is in an invalid or error state. 1135 * This may happen if another detector has been instantiated or the 1136 * {@link VoiceInteractionService} hosting this detector has been shut down. 1137 */ 1138 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) startRecognition(@ecognitionFlags int recognitionFlags, @NonNull byte[] data)1139 public boolean startRecognition(@RecognitionFlags int recognitionFlags, @NonNull byte[] data) { 1140 synchronized (mLock) { 1141 return startRecognitionLocked(recognitionFlags, data) 1142 == STATUS_OK; 1143 } 1144 } 1145 1146 /** 1147 * Starts recognition for the associated keyphrase. 1148 * Caller must be the active voice interaction service via 1149 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1150 * 1151 * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO 1152 * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS 1153 * 1154 * @param recognitionFlags The flags to control the recognition properties. 1155 * @return Indicates whether the call succeeded or not. 1156 * @throws UnsupportedOperationException if the recognition isn't supported. 1157 * Callers should only call this method after a supported state callback on 1158 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1159 * @throws IllegalStateException if the detector is in an invalid or error state. 1160 * This may happen if another detector has been instantiated or the 1161 * {@link VoiceInteractionService} hosting this detector has been shut down. 1162 */ 1163 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) startRecognition(@ecognitionFlags int recognitionFlags)1164 public boolean startRecognition(@RecognitionFlags int recognitionFlags) { 1165 if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")"); 1166 synchronized (mLock) { 1167 return startRecognitionLocked(recognitionFlags, /* data= */new byte[0]) == STATUS_OK; 1168 } 1169 } 1170 1171 /** 1172 * Starts recognition for the associated keyphrase. 1173 * 1174 * @see #startRecognition(int) 1175 */ 1176 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 1177 @Override startRecognition()1178 public boolean startRecognition() { 1179 return startRecognition(0); 1180 } 1181 1182 /** 1183 * Stops recognition for the associated keyphrase. 1184 * Caller must be the active voice interaction service via 1185 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1186 * 1187 * @return Indicates whether the call succeeded or not. 1188 * @throws UnsupportedOperationException if the recognition isn't supported. 1189 * Callers should only call this method after a supported state callback on 1190 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1191 * @throws IllegalStateException if the detector is in an invalid or error state. 1192 * This may happen if another detector has been instantiated or the 1193 * {@link VoiceInteractionService} hosting this detector has been shut down. 1194 */ 1195 // TODO: Remove this RequiresPermission since it isn't actually enforced. Also fix the javadoc 1196 // about permissions enforcement (when it throws vs when it just returns false) for other 1197 // methods in this class. 1198 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 1199 @Override stopRecognition()1200 public boolean stopRecognition() { 1201 if (DBG) Slog.d(TAG, "stopRecognition()"); 1202 synchronized (mLock) { 1203 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1204 throw new IllegalStateException( 1205 "stopRecognition called on an invalid detector or error state"); 1206 } 1207 1208 // Check if we can start/stop a recognition. 1209 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 1210 throw new UnsupportedOperationException( 1211 "Recognition for the given keyphrase is not supported"); 1212 } 1213 1214 return stopRecognitionLocked() == STATUS_OK; 1215 } 1216 } 1217 1218 /** 1219 * Set a model specific {@link ModelParams} with the given value. This 1220 * parameter will keep its value for the duration the model is loaded regardless of starting and 1221 * stopping recognition. Once the model is unloaded, the value will be lost. 1222 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before calling this 1223 * method. 1224 * Caller must be the active voice interaction service via 1225 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1226 * 1227 * @param modelParam {@link ModelParams} 1228 * @param value Value to set 1229 * @return - {@link SoundTrigger#STATUS_OK} in case of success 1230 * - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached 1231 * - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter 1232 * - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or 1233 * if API is not supported by HAL 1234 * @throws IllegalStateException if the detector is in an invalid or error state. 1235 * This may happen if another detector has been instantiated or the 1236 * {@link VoiceInteractionService} hosting this detector has been shut down. 1237 */ 1238 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) setParameter(@odelParams int modelParam, int value)1239 public int setParameter(@ModelParams int modelParam, int value) { 1240 if (DBG) { 1241 Slog.d(TAG, "setParameter(" + modelParam + ", " + value + ")"); 1242 } 1243 1244 synchronized (mLock) { 1245 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1246 throw new IllegalStateException( 1247 "setParameter called on an invalid detector or error state"); 1248 } 1249 1250 return setParameterLocked(modelParam, value); 1251 } 1252 } 1253 1254 /** 1255 * Get a model specific {@link ModelParams}. This parameter will keep its value 1256 * for the duration the model is loaded regardless of starting and stopping recognition. 1257 * Once the model is unloaded, the value will be lost. If the value is not set, a default 1258 * value is returned. See {@link ModelParams} for parameter default values. 1259 * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before 1260 * calling this method. 1261 * Caller must be the active voice interaction service via 1262 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1263 * 1264 * @param modelParam {@link ModelParams} 1265 * @return value of parameter 1266 * @throws IllegalStateException if the detector is in an invalid or error state. 1267 * This may happen if another detector has been instantiated or the 1268 * {@link VoiceInteractionService} hosting this detector has been shut down. 1269 */ 1270 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) getParameter(@odelParams int modelParam)1271 public int getParameter(@ModelParams int modelParam) { 1272 if (DBG) { 1273 Slog.d(TAG, "getParameter(" + modelParam + ")"); 1274 } 1275 1276 synchronized (mLock) { 1277 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1278 throw new IllegalStateException( 1279 "getParameter called on an invalid detector or error state"); 1280 } 1281 1282 return getParameterLocked(modelParam); 1283 } 1284 } 1285 1286 /** 1287 * Determine if parameter control is supported for the given model handle. 1288 * This method should be checked prior to calling {@link AlwaysOnHotwordDetector#setParameter} 1289 * or {@link AlwaysOnHotwordDetector#getParameter}. 1290 * Caller must be the active voice interaction service via 1291 * Settings.Secure.VOICE_INTERACTION_SERVICE. 1292 * 1293 * @param modelParam {@link ModelParams} 1294 * @return supported range of parameter, null if not supported 1295 * @throws IllegalStateException if the detector is in an invalid or error state. 1296 * This may happen if another detector has been instantiated or the 1297 * {@link VoiceInteractionService} hosting this detector has been shut down. 1298 */ 1299 @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD}) 1300 @Nullable queryParameter(@odelParams int modelParam)1301 public ModelParamRange queryParameter(@ModelParams int modelParam) { 1302 if (DBG) { 1303 Slog.d(TAG, "queryParameter(" + modelParam + ")"); 1304 } 1305 1306 synchronized (mLock) { 1307 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1308 throw new IllegalStateException( 1309 "queryParameter called on an invalid detector or error state"); 1310 } 1311 1312 return queryParameterLocked(modelParam); 1313 } 1314 } 1315 1316 /** 1317 * Creates an intent to start the enrollment for the associated keyphrase. 1318 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1319 * Starting re-enrollment is only valid if the keyphrase is un-enrolled, 1320 * i.e. {@link #STATE_KEYPHRASE_UNENROLLED}, 1321 * otherwise {@link #createReEnrollIntent()} should be preferred. 1322 * 1323 * @return An {@link Intent} to start enrollment for the given keyphrase. 1324 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1325 * Callers should only call this method after a supported state callback on 1326 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1327 * @throws IllegalStateException if the detector is in an invalid state. 1328 * This may happen if another detector has been instantiated or the 1329 * {@link VoiceInteractionService} hosting this detector has been shut down. 1330 */ 1331 @Nullable createEnrollIntent()1332 public Intent createEnrollIntent() { 1333 if (DBG) Slog.d(TAG, "createEnrollIntent"); 1334 synchronized (mLock) { 1335 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_ENROLL); 1336 } 1337 } 1338 1339 /** 1340 * Creates an intent to start the un-enrollment for the associated keyphrase. 1341 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1342 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 1343 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 1344 * 1345 * @return An {@link Intent} to start un-enrollment for the given keyphrase. 1346 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1347 * Callers should only call this method after a supported state callback on 1348 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1349 * @throws IllegalStateException if the detector is in an invalid state. 1350 * This may happen if another detector has been instantiated or the 1351 * {@link VoiceInteractionService} hosting this detector has been shut down. 1352 */ 1353 @Nullable createUnEnrollIntent()1354 public Intent createUnEnrollIntent() { 1355 if (DBG) Slog.d(TAG, "createUnEnrollIntent"); 1356 synchronized (mLock) { 1357 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_UN_ENROLL); 1358 } 1359 } 1360 1361 /** 1362 * Creates an intent to start the re-enrollment for the associated keyphrase. 1363 * This intent must be invoked using {@link Context#startForegroundService(Intent)}. 1364 * Starting re-enrollment is only valid if the keyphrase is already enrolled, 1365 * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error. 1366 * 1367 * @return An {@link Intent} to start re-enrollment for the given keyphrase. 1368 * @throws UnsupportedOperationException if managing they keyphrase isn't supported. 1369 * Callers should only call this method after a supported state callback on 1370 * {@link Callback#onAvailabilityChanged(int)} to avoid this exception. 1371 * @throws IllegalStateException if the detector is in an invalid or error state. 1372 * This may happen if another detector has been instantiated or the 1373 * {@link VoiceInteractionService} hosting this detector has been shut down. 1374 */ 1375 @Nullable createReEnrollIntent()1376 public Intent createReEnrollIntent() { 1377 if (DBG) Slog.d(TAG, "createReEnrollIntent"); 1378 synchronized (mLock) { 1379 return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_RE_ENROLL); 1380 } 1381 } 1382 1383 @GuardedBy("mLock") getManageIntentLocked(@eyphraseEnrollmentInfo.ManageActions int action)1384 private Intent getManageIntentLocked(@KeyphraseEnrollmentInfo.ManageActions int action) { 1385 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1386 throw new IllegalStateException( 1387 "getManageIntent called on an invalid detector or error state"); 1388 } 1389 1390 // This method only makes sense if we can actually support a recognition. 1391 if (mAvailability != STATE_KEYPHRASE_ENROLLED 1392 && mAvailability != STATE_KEYPHRASE_UNENROLLED) { 1393 throw new UnsupportedOperationException( 1394 "Managing the given keyphrase is not supported"); 1395 } 1396 1397 return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); 1398 } 1399 1400 /** {@inheritDoc} */ 1401 @Override destroy()1402 public void destroy() { 1403 synchronized (mLock) { 1404 detachSessionLocked(); 1405 1406 mAvailability = STATE_INVALID; 1407 mIsAvailabilityOverriddenByTestApi = false; 1408 notifyStateChangedLocked(); 1409 } 1410 super.destroy(); 1411 } 1412 detachSessionLocked()1413 private void detachSessionLocked() { 1414 try { 1415 if (DBG) Slog.d(TAG, "detachSessionLocked() " + mSoundTriggerSession); 1416 if (mSoundTriggerSession != null) { 1417 mSoundTriggerSession.detach(); 1418 } 1419 } catch (RemoteException e) { 1420 e.rethrowFromSystemServer(); 1421 } 1422 } 1423 1424 /** 1425 * @hide 1426 */ 1427 @Override isUsingSandboxedDetectionService()1428 public boolean isUsingSandboxedDetectionService() { 1429 return mSupportSandboxedDetectionService; 1430 } 1431 1432 /** 1433 * Reloads the sound models from the service. 1434 * 1435 * @hide 1436 */ 1437 // TODO(b/281608561): remove the enrollment flow from AlwaysOnHotwordDetector onSoundModelsChanged()1438 void onSoundModelsChanged() { 1439 synchronized (mLock) { 1440 if (mAvailability == STATE_INVALID 1441 || mAvailability == STATE_HARDWARE_UNAVAILABLE 1442 || mAvailability == STATE_ERROR) { 1443 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config" 1444 + " or in the error state"); 1445 return; 1446 } 1447 1448 // Because this method reflects an update from the system service models, we should not 1449 // update the client of an availability change when the availability has been overridden 1450 // via a test API. 1451 if (mIsAvailabilityOverriddenByTestApi) { 1452 Slog.w(TAG, "Suppressing system availability update. " 1453 + "Availability is overridden by test API."); 1454 return; 1455 } 1456 1457 // Stop the recognition before proceeding if we are in the enrolled state. 1458 // The framework makes the guarantee that an actively used model is present in the 1459 // system server's enrollment database. For this reason we much stop an actively running 1460 // model when the underlying sound model in enrollment database no longer match. 1461 if (mAvailability == STATE_KEYPHRASE_ENROLLED) { 1462 // A SoundTriggerFailure will be sent to the client if the model state was 1463 // changed. This is an overloading of the onFailure usage because we are sending a 1464 // callback even in the successful stop case. If stopRecognition is successful, 1465 // suggested next action RESTART_RECOGNITION will be sent. 1466 // TODO(b/281608561): This code path will be removed with other enrollment flows in 1467 // this class. 1468 try { 1469 int result = stopRecognitionLocked(); 1470 if (result == STATUS_OK) { 1471 sendSoundTriggerFailure(new SoundTriggerFailure(ERROR_CODE_UNKNOWN, 1472 "stopped recognition because of enrollment update", 1473 FailureSuggestedAction.RESTART_RECOGNITION)); 1474 } 1475 // only log to logcat here because many failures can be false positives such as 1476 // calling stopRecognition where there is no started session. 1477 Log.w(TAG, "Failed to stop recognition after enrollment update: code=" 1478 + result); 1479 } catch (Exception e) { 1480 Slog.w(TAG, "Failed to stop recognition after enrollment update", e); 1481 if (CompatChanges.isChangeEnabled(SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS)) { 1482 sendSoundTriggerFailure(new SoundTriggerFailure(ERROR_CODE_UNKNOWN, 1483 "Failed to stop recognition after enrollment update: " 1484 + Log.getStackTraceString(e), 1485 FailureSuggestedAction.RECREATE_DETECTOR)); 1486 } else { 1487 updateAndNotifyStateChangedLocked(STATE_ERROR); 1488 } 1489 return; 1490 } 1491 } 1492 1493 // Execute a refresh availability task - which should then notify of a change. 1494 new RefreshAvailabilityTask().execute(); 1495 } 1496 } 1497 1498 @GuardedBy("mLock") 1499 @SuppressWarnings("FlaggedApi") // RecognitionConfig.Builder is available internally. startRecognitionLocked(int recognitionFlags, @NonNull byte[] data)1500 private int startRecognitionLocked(int recognitionFlags, @NonNull byte[] data) { 1501 if (DBG) { 1502 Slog.d(TAG, "startRecognition(" 1503 + recognitionFlags 1504 + ", " + Arrays.toString(data) + ")"); 1505 } 1506 if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) { 1507 throw new IllegalStateException( 1508 "startRecognition called on an invalid detector or error state"); 1509 } 1510 1511 // Check if we can start/stop a recognition. 1512 if (mAvailability != STATE_KEYPHRASE_ENROLLED) { 1513 throw new UnsupportedOperationException( 1514 "Recognition for the given keyphrase is not supported"); 1515 } 1516 1517 List<KeyphraseRecognitionExtra> recognitionExtra = 1518 new ArrayList<KeyphraseRecognitionExtra>(1); 1519 // TODO: Do we need to do something about the confidence level here? 1520 recognitionExtra.add(new KeyphraseRecognitionExtra(mKeyphraseMetadata.getId(), 1521 mKeyphraseMetadata.getRecognitionModeFlags(), 0, new ConfidenceLevel[0])); 1522 boolean captureTriggerAudio = 1523 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; 1524 boolean allowMultipleTriggers = 1525 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0; 1526 boolean runInBatterySaver = (recognitionFlags&RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER) != 0; 1527 1528 int audioCapabilities = 0; 1529 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION) != 0) { 1530 audioCapabilities |= AUDIO_CAPABILITY_ECHO_CANCELLATION; 1531 } 1532 if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION) != 0) { 1533 audioCapabilities |= AUDIO_CAPABILITY_NOISE_SUPPRESSION; 1534 } 1535 1536 int code; 1537 try { 1538 code = mSoundTriggerSession.startRecognition( 1539 mKeyphraseMetadata.getId(), 1540 mLocale.toLanguageTag(), 1541 mInternalCallback, 1542 new RecognitionConfig.Builder() 1543 .setCaptureRequested(captureTriggerAudio) 1544 .setMultipleTriggersAllowed(allowMultipleTriggers) 1545 .setKeyphrases(recognitionExtra) 1546 .setData(data) 1547 .setAudioCapabilities(audioCapabilities) 1548 .build(), 1549 runInBatterySaver); 1550 } catch (RemoteException e) { 1551 throw e.rethrowFromSystemServer(); 1552 } 1553 1554 if (code != STATUS_OK) { 1555 Slog.w(TAG, "startRecognition() failed with error code " + code); 1556 } 1557 return code; 1558 } 1559 1560 @GuardedBy("mLock") stopRecognitionLocked()1561 private int stopRecognitionLocked() { 1562 int code; 1563 try { 1564 code = mSoundTriggerSession.stopRecognition(mKeyphraseMetadata.getId(), 1565 mInternalCallback); 1566 } catch (RemoteException e) { 1567 throw e.rethrowFromSystemServer(); 1568 } 1569 1570 if (code != STATUS_OK) { 1571 Slog.w(TAG, "stopRecognition() failed with error code " + code); 1572 } 1573 return code; 1574 } 1575 1576 @GuardedBy("mLock") setParameterLocked(@odelParams int modelParam, int value)1577 private int setParameterLocked(@ModelParams int modelParam, int value) { 1578 try { 1579 int code = mSoundTriggerSession.setParameter(mKeyphraseMetadata.getId(), modelParam, 1580 value); 1581 1582 if (code != STATUS_OK) { 1583 Slog.w(TAG, "setParameter failed with error code " + code); 1584 } 1585 1586 return code; 1587 } catch (RemoteException e) { 1588 throw e.rethrowFromSystemServer(); 1589 } 1590 } 1591 1592 @GuardedBy("mLock") getParameterLocked(@odelParams int modelParam)1593 private int getParameterLocked(@ModelParams int modelParam) { 1594 try { 1595 return mSoundTriggerSession.getParameter(mKeyphraseMetadata.getId(), modelParam); 1596 } catch (RemoteException e) { 1597 throw e.rethrowFromSystemServer(); 1598 } 1599 } 1600 1601 @GuardedBy("mLock") 1602 @Nullable queryParameterLocked(@odelParams int modelParam)1603 private ModelParamRange queryParameterLocked(@ModelParams int modelParam) { 1604 try { 1605 SoundTrigger.ModelParamRange modelParamRange = 1606 mSoundTriggerSession.queryParameter(mKeyphraseMetadata.getId(), modelParam); 1607 1608 if (modelParamRange == null) { 1609 return null; 1610 } 1611 1612 return new ModelParamRange(modelParamRange); 1613 } catch (RemoteException e) { 1614 throw e.rethrowFromSystemServer(); 1615 } 1616 } 1617 1618 @GuardedBy("mLock") updateAndNotifyStateChangedLocked(int availability)1619 private void updateAndNotifyStateChangedLocked(int availability) { 1620 updateAvailabilityLocked(availability); 1621 notifyStateChangedLocked(); 1622 } 1623 1624 @GuardedBy("mLock") updateAvailabilityLocked(int availability)1625 private void updateAvailabilityLocked(int availability) { 1626 if (DBG) { 1627 Slog.d(TAG, "Hotword availability changed from " + mAvailability 1628 + " -> " + availability); 1629 } 1630 if (!mIsAvailabilityOverriddenByTestApi) { 1631 mAvailability = availability; 1632 } 1633 } 1634 1635 @GuardedBy("mLock") notifyStateChangedLocked()1636 private void notifyStateChangedLocked() { 1637 Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED); 1638 message.arg1 = mAvailability; 1639 message.sendToTarget(); 1640 } 1641 1642 @GuardedBy("mLock") sendUnknownFailure(String failureMessage)1643 private void sendUnknownFailure(String failureMessage) { 1644 // update but do not call onAvailabilityChanged callback for STATE_ERROR 1645 updateAvailabilityLocked(STATE_ERROR); 1646 Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE, failureMessage).sendToTarget(); 1647 } 1648 sendSoundTriggerFailure(@onNull SoundTriggerFailure soundTriggerFailure)1649 private void sendSoundTriggerFailure(@NonNull SoundTriggerFailure soundTriggerFailure) { 1650 Message.obtain(mHandler, MSG_DETECTION_SOUND_TRIGGER_FAILURE, soundTriggerFailure) 1651 .sendToTarget(); 1652 } 1653 1654 /** @hide */ 1655 static final class SoundTriggerListener extends IHotwordRecognitionStatusCallback.Stub { 1656 private final Handler mHandler; 1657 SoundTriggerListener(Handler handler)1658 public SoundTriggerListener(Handler handler) { 1659 mHandler = handler; 1660 } 1661 1662 @Override onKeyphraseDetected( KeyphraseRecognitionEvent event, HotwordDetectedResult result)1663 public void onKeyphraseDetected( 1664 KeyphraseRecognitionEvent event, HotwordDetectedResult result) { 1665 if (DBG) { 1666 Slog.d(TAG, "onDetected(" + event + ")"); 1667 } else { 1668 Slog.i(TAG, "onDetected"); 1669 } 1670 Message.obtain(mHandler, MSG_HOTWORD_DETECTED, 1671 new EventPayload.Builder(event) 1672 .setHotwordDetectedResult(result) 1673 .build()) 1674 .sendToTarget(); 1675 } 1676 1677 @Override onKeyphraseDetectedFromExternalSource(HotwordDetectedResult result)1678 public void onKeyphraseDetectedFromExternalSource(HotwordDetectedResult result) { 1679 Slog.i(TAG, "onKeyphraseDetectedFromExternalSource"); 1680 EventPayload.Builder eventPayloadBuilder = new EventPayload.Builder(); 1681 if (android.app.wearable.Flags.enableHotwordWearableSensingApi()) { 1682 eventPayloadBuilder.setIsRecognitionStopped(false); 1683 } 1684 Message.obtain( 1685 mHandler, 1686 MSG_HOTWORD_DETECTED, 1687 eventPayloadBuilder.setHotwordDetectedResult(result).build()) 1688 .sendToTarget(); 1689 } 1690 1691 @Override onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)1692 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 1693 Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event); 1694 } 1695 1696 @Override onRejected(@onNull HotwordRejectedResult result)1697 public void onRejected(@NonNull HotwordRejectedResult result) { 1698 if (DBG) { 1699 Slog.d(TAG, "onRejected(" + result + ")"); 1700 } else { 1701 Slog.i(TAG, "onRejected"); 1702 } 1703 Message.obtain(mHandler, MSG_HOTWORD_REJECTED, result).sendToTarget(); 1704 } 1705 1706 @Override onHotwordDetectionServiceFailure( HotwordDetectionServiceFailure hotwordDetectionServiceFailure)1707 public void onHotwordDetectionServiceFailure( 1708 HotwordDetectionServiceFailure hotwordDetectionServiceFailure) { 1709 Slog.v(TAG, "onHotwordDetectionServiceFailure: " + hotwordDetectionServiceFailure); 1710 if (hotwordDetectionServiceFailure != null) { 1711 Message.obtain(mHandler, MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE, 1712 hotwordDetectionServiceFailure).sendToTarget(); 1713 } else { 1714 Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE, 1715 "Error data is null").sendToTarget(); 1716 } 1717 } 1718 1719 @Override onVisualQueryDetectionServiceFailure( VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure)1720 public void onVisualQueryDetectionServiceFailure( 1721 VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure) 1722 throws RemoteException { 1723 // It should never be called here. 1724 Slog.w(TAG, 1725 "onVisualQueryDetectionServiceFailure: " + visualQueryDetectionServiceFailure); 1726 } 1727 1728 @Override onSoundTriggerFailure(SoundTriggerFailure soundTriggerFailure)1729 public void onSoundTriggerFailure(SoundTriggerFailure soundTriggerFailure) { 1730 Message.obtain(mHandler, MSG_DETECTION_SOUND_TRIGGER_FAILURE, 1731 Objects.requireNonNull(soundTriggerFailure)).sendToTarget(); 1732 } 1733 1734 @Override onUnknownFailure(String errorMessage)1735 public void onUnknownFailure(String errorMessage) throws RemoteException { 1736 Slog.v(TAG, "onUnknownFailure: " + errorMessage); 1737 Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE, 1738 !TextUtils.isEmpty(errorMessage) ? errorMessage 1739 : "Error data is null").sendToTarget(); 1740 } 1741 1742 @Override onRecognitionPaused()1743 public void onRecognitionPaused() { 1744 Slog.i(TAG, "onRecognitionPaused"); 1745 mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE); 1746 } 1747 1748 @Override onRecognitionResumed()1749 public void onRecognitionResumed() { 1750 Slog.i(TAG, "onRecognitionResumed"); 1751 mHandler.sendEmptyMessage(MSG_DETECTION_RESUME); 1752 } 1753 1754 @Override onStatusReported(int status)1755 public void onStatusReported(int status) { 1756 if (DBG) { 1757 Slog.d(TAG, "onStatusReported(" + status + ")"); 1758 } else { 1759 Slog.i(TAG, "onStatusReported"); 1760 } 1761 Message message = Message.obtain(mHandler, MSG_HOTWORD_STATUS_REPORTED); 1762 message.arg1 = status; 1763 message.sendToTarget(); 1764 } 1765 1766 @Override onProcessRestarted()1767 public void onProcessRestarted() { 1768 Slog.i(TAG, "onProcessRestarted"); 1769 mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED); 1770 } 1771 1772 @Override onOpenFile(String filename, AndroidFuture future)1773 public void onOpenFile(String filename, AndroidFuture future) throws RemoteException { 1774 throw new UnsupportedOperationException("Hotword cannot access files from the disk."); 1775 } 1776 } 1777 onDetectorRemoteException()1778 void onDetectorRemoteException() { 1779 Message.obtain(mHandler, MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE, 1780 new HotwordDetectionServiceFailure( 1781 HotwordDetectionServiceFailure.ERROR_CODE_REMOTE_EXCEPTION, 1782 "Detector remote exception occurs")).sendToTarget(); 1783 } 1784 1785 class MyHandler extends Handler { MyHandler(@onNull Looper looper)1786 MyHandler(@NonNull Looper looper) { 1787 super(looper); 1788 } 1789 1790 @Override handleMessage(Message msg)1791 public void handleMessage(Message msg) { 1792 synchronized (mLock) { 1793 if (mAvailability == STATE_INVALID) { 1794 Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector"); 1795 return; 1796 } 1797 } 1798 final Message message = Message.obtain(msg); 1799 Binder.withCleanCallingIdentity(() -> mExternalExecutor.execute(() -> { 1800 Slog.i(TAG, "handle message " + message.what); 1801 switch (message.what) { 1802 case MSG_AVAILABILITY_CHANGED: 1803 mExternalCallback.onAvailabilityChanged(message.arg1); 1804 break; 1805 case MSG_HOTWORD_DETECTED: 1806 mExternalCallback.onDetected((EventPayload) message.obj); 1807 break; 1808 case MSG_DETECTION_ERROR: 1809 // TODO(b/271534248): After reverting the workaround, this logic is still 1810 // necessary. 1811 mExternalCallback.onError(); 1812 break; 1813 case MSG_DETECTION_PAUSE: 1814 mExternalCallback.onRecognitionPaused(); 1815 break; 1816 case MSG_DETECTION_RESUME: 1817 mExternalCallback.onRecognitionResumed(); 1818 break; 1819 case MSG_HOTWORD_REJECTED: 1820 mExternalCallback.onRejected((HotwordRejectedResult) message.obj); 1821 break; 1822 case MSG_HOTWORD_STATUS_REPORTED: 1823 mExternalCallback.onHotwordDetectionServiceInitialized(message.arg1); 1824 break; 1825 case MSG_PROCESS_RESTARTED: 1826 mExternalCallback.onHotwordDetectionServiceRestarted(); 1827 break; 1828 case MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE: 1829 mExternalCallback.onFailure((HotwordDetectionServiceFailure) message.obj); 1830 break; 1831 case MSG_DETECTION_SOUND_TRIGGER_FAILURE: 1832 mExternalCallback.onFailure((SoundTriggerFailure) message.obj); 1833 break; 1834 case MSG_DETECTION_UNKNOWN_FAILURE: 1835 mExternalCallback.onUnknownFailure((String) message.obj); 1836 break; 1837 default: 1838 super.handleMessage(message); 1839 } 1840 message.recycle(); 1841 })); 1842 } 1843 } 1844 1845 // TODO(b/267681692): remove the AsyncTask usage 1846 class RefreshAvailabilityTask extends AsyncTask<Void, Void, Void> { 1847 1848 @Override doInBackground(Void... params)1849 public Void doInBackground(Void... params) { 1850 try { 1851 int availability = internalGetInitialAvailability(); 1852 1853 synchronized (mLock) { 1854 if (availability == STATE_NOT_READY) { 1855 internalUpdateEnrolledKeyphraseMetadata(); 1856 if (mKeyphraseMetadata != null) { 1857 availability = STATE_KEYPHRASE_ENROLLED; 1858 } else { 1859 availability = STATE_KEYPHRASE_UNENROLLED; 1860 } 1861 } 1862 updateAndNotifyStateChangedLocked(availability); 1863 } 1864 } catch (Exception e) { 1865 // Any exception here not caught will crash the process because AsyncTask does not 1866 // bubble up the exceptions to the client app, so we must propagate it to the app. 1867 Slog.w(TAG, "Failed to refresh availability", e); 1868 synchronized (mLock) { 1869 if (CompatChanges.isChangeEnabled(SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS)) { 1870 sendUnknownFailure( 1871 "Failed to refresh availability: " + Log.getStackTraceString(e)); 1872 } else { 1873 updateAndNotifyStateChangedLocked(STATE_ERROR); 1874 } 1875 } 1876 } 1877 1878 return null; 1879 } 1880 1881 /** 1882 * @return The initial availability without checking the enrollment status. 1883 */ internalGetInitialAvailability()1884 private int internalGetInitialAvailability() { 1885 synchronized (mLock) { 1886 // This detector has already been invalidated. 1887 if (mAvailability == STATE_INVALID) { 1888 return STATE_INVALID; 1889 } 1890 } 1891 1892 if (!CompatChanges.isChangeEnabled(THROW_ON_INITIALIZE_IF_NO_DSP)) { 1893 ModuleProperties dspModuleProperties; 1894 try { 1895 dspModuleProperties = 1896 mSoundTriggerSession.getDspModuleProperties(); 1897 } catch (RemoteException e) { 1898 throw e.rethrowFromSystemServer(); 1899 } 1900 1901 // No DSP available 1902 if (dspModuleProperties == null) { 1903 return STATE_HARDWARE_UNAVAILABLE; 1904 } 1905 } 1906 1907 return STATE_NOT_READY; 1908 } 1909 internalUpdateEnrolledKeyphraseMetadata()1910 private void internalUpdateEnrolledKeyphraseMetadata() { 1911 try { 1912 mKeyphraseMetadata = mModelManagementService.getEnrolledKeyphraseMetadata( 1913 mText, mLocale.toLanguageTag()); 1914 } catch (RemoteException e) { 1915 throw e.rethrowFromSystemServer(); 1916 } 1917 } 1918 } 1919 1920 @Override equals(Object obj)1921 public boolean equals(Object obj) { 1922 if (CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) { 1923 if (!(obj instanceof AlwaysOnHotwordDetector)) { 1924 return false; 1925 } 1926 AlwaysOnHotwordDetector other = (AlwaysOnHotwordDetector) obj; 1927 return TextUtils.equals(mText, other.mText) && mLocale.equals(other.mLocale); 1928 } 1929 1930 return super.equals(obj); 1931 } 1932 1933 @Override hashCode()1934 public int hashCode() { 1935 return Objects.hash(mText, mLocale); 1936 } 1937 1938 /** @hide */ 1939 @Override dump(String prefix, PrintWriter pw)1940 public void dump(String prefix, PrintWriter pw) { 1941 synchronized (mLock) { 1942 pw.print(prefix); pw.print("Text="); pw.println(mText); 1943 pw.print(prefix); pw.print("Locale="); pw.println(mLocale); 1944 pw.print(prefix); pw.print("Availability="); pw.println(mAvailability); 1945 pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata); 1946 pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo); 1947 } 1948 } 1949 } 1950