• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.service.voice;
18 
19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD;
20 import static android.Manifest.permission.RECORD_AUDIO;
21 import static android.service.voice.SoundTriggerFailure.ERROR_CODE_UNKNOWN;
22 import static android.service.voice.VoiceInteractionService.MULTIPLE_ACTIVE_HOTWORD_DETECTORS;
23 
24 import android.annotation.ElapsedRealtimeLong;
25 import android.annotation.FlaggedApi;
26 import android.annotation.IntDef;
27 import android.annotation.NonNull;
28 import android.annotation.Nullable;
29 import android.annotation.RequiresPermission;
30 import android.annotation.SuppressLint;
31 import android.annotation.SystemApi;
32 import android.annotation.TestApi;
33 import android.app.ActivityThread;
34 import android.app.compat.CompatChanges;
35 import android.compat.annotation.ChangeId;
36 import android.compat.annotation.EnabledSince;
37 import android.compat.annotation.UnsupportedAppUsage;
38 import android.content.Context;
39 import android.content.Intent;
40 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo;
41 import android.hardware.soundtrigger.KeyphraseMetadata;
42 import android.hardware.soundtrigger.SoundTrigger;
43 import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel;
44 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent;
45 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;
46 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
47 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
48 import android.media.AudioFormat;
49 import android.media.permission.Identity;
50 import android.os.AsyncTask;
51 import android.os.Binder;
52 import android.os.Build;
53 import android.os.Handler;
54 import android.os.HandlerExecutor;
55 import android.os.IBinder;
56 import android.os.Looper;
57 import android.os.Message;
58 import android.os.ParcelFileDescriptor;
59 import android.os.PersistableBundle;
60 import android.os.RemoteException;
61 import android.os.SharedMemory;
62 import android.os.SystemClock;
63 import android.text.TextUtils;
64 import android.util.Log;
65 import android.util.Slog;
66 
67 import com.android.internal.annotations.GuardedBy;
68 import com.android.internal.app.IHotwordRecognitionStatusCallback;
69 import com.android.internal.app.IVoiceInteractionManagerService;
70 import com.android.internal.app.IVoiceInteractionSoundTriggerSession;
71 import com.android.internal.infra.AndroidFuture;
72 
73 import java.io.PrintWriter;
74 import java.lang.annotation.Retention;
75 import java.lang.annotation.RetentionPolicy;
76 import java.util.ArrayList;
77 import java.util.Arrays;
78 import java.util.Collections;
79 import java.util.HashSet;
80 import java.util.List;
81 import java.util.Locale;
82 import java.util.Objects;
83 import java.util.Set;
84 import java.util.concurrent.Executor;
85 
86 /**
87  * A class that lets a VoiceInteractionService implementation interact with
88  * always-on keyphrase detection APIs.
89  *
90  * @hide
91  * TODO(b/168605867): Once Metalava supports expressing a removed public, but current system API,
92  *                    mark and track it as such.
93  */
94 @SystemApi
95 public class AlwaysOnHotwordDetector extends AbstractDetector {
96     //---- States of Keyphrase availability. Return codes for onAvailabilityChanged() ----//
97     /**
98      * Indicates that this hotword detector is no longer valid for any recognition
99      * and should not be used anymore.
100      */
101     private static final int STATE_INVALID = -3;
102 
103     /**
104      * Indicates that recognition for the given keyphrase is not available on the system
105      * because of the hardware configuration.
106      * No further interaction should be performed with the detector that returns this availability.
107      */
108     public static final int STATE_HARDWARE_UNAVAILABLE = -2;
109 
110     /**
111      * Indicates that recognition for the given keyphrase is not supported.
112      * No further interaction should be performed with the detector that returns this availability.
113      *
114      * @deprecated This is no longer a valid state. Enrollment can occur outside of
115      * {@link KeyphraseEnrollmentInfo} through another privileged application. We can no longer
116      * determine ahead of time if the keyphrase and locale are unsupported by the system.
117      */
118     @Deprecated
119     public static final int STATE_KEYPHRASE_UNSUPPORTED = -1;
120 
121     /**
122      * Indicates that the given keyphrase is not enrolled.
123      * The caller may choose to begin an enrollment flow for the keyphrase.
124      */
125     public static final int STATE_KEYPHRASE_UNENROLLED = 1;
126 
127     /**
128      * Indicates that the given keyphrase is currently enrolled and it's possible to start
129      * recognition for it.
130      */
131     public static final int STATE_KEYPHRASE_ENROLLED = 2;
132 
133     /**
134      * Indicates that the availability state of the active keyphrase can't be known due to an error.
135      *
136      * <p>NOTE: No further interaction should be performed with the detector that returns this
137      * state, it would be better to create {@link AlwaysOnHotwordDetector} again.
138      */
139     public static final int STATE_ERROR = 3;
140 
141     /**
142      * Indicates that the detector isn't ready currently.
143      */
144     private static final int STATE_NOT_READY = 0;
145 
146     //-- Flags for startRecognition    ----//
147     /** @hide */
148     @Retention(RetentionPolicy.SOURCE)
149     @IntDef(flag = true, prefix = { "RECOGNITION_FLAG_" }, value = {
150             RECOGNITION_FLAG_NONE,
151             RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO,
152             RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS,
153             RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION,
154             RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION,
155             RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER,
156     })
157     public @interface RecognitionFlags {}
158 
159     /**
160      * Empty flag for {@link #startRecognition(int)}.
161      *
162      * @hide
163      */
164     public static final int RECOGNITION_FLAG_NONE = 0;
165 
166     /**
167      * Recognition flag for {@link #startRecognition(int)} that indicates
168      * whether the trigger audio for hotword needs to be captured.
169      */
170     public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1;
171 
172     /**
173      * Recognition flag for {@link #startRecognition(int)} that indicates
174      * whether the recognition should keep going on even after the keyphrase triggers.
175      * If this flag is specified, it's possible to get multiple triggers after a
176      * call to {@link #startRecognition(int)} if the user speaks the keyphrase multiple times.
177      * When this isn't specified, the default behavior is to stop recognition once the
178      * keyphrase is spoken, till the caller starts recognition again.
179      */
180     public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2;
181 
182     /**
183      * Audio capabilities flag for {@link #startRecognition(int)} that indicates
184      * if the underlying recognition should use AEC.
185      * This capability may or may not be supported by the system, and support can be queried
186      * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for
187      * this flag is {@link #AUDIO_CAPABILITY_ECHO_CANCELLATION}. If this flag is passed without the
188      * audio capability supported, there will be no audio effect applied.
189      */
190     public static final int RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION = 0x4;
191 
192     /**
193      * Audio capabilities flag for {@link #startRecognition(int)} that indicates
194      * if the underlying recognition should use noise suppression.
195      * This capability may or may not be supported by the system, and support can be queried
196      * by calling {@link #getSupportedAudioCapabilities()}. The corresponding capabilities field for
197      * this flag is {@link #AUDIO_CAPABILITY_NOISE_SUPPRESSION}. If this flag is passed without the
198      * audio capability supported, there will be no audio effect applied.
199      */
200     public static final int RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION = 0x8;
201 
202     /**
203      * Recognition flag for {@link #startRecognition(int)} that indicates whether the recognition
204      * should continue after battery saver mode is enabled.
205      * When this flag is specified, the caller will be checked for
206      * {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER} permission granted.
207      */
208     public static final int RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER = 0x10;
209 
210     //---- Recognition mode flags. Return codes for getSupportedRecognitionModes() ----//
211     // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags.
212 
213     /** @hide */
214     @Retention(RetentionPolicy.SOURCE)
215     @IntDef(flag = true, prefix = { "RECOGNITION_MODE_" }, value = {
216             RECOGNITION_MODE_VOICE_TRIGGER,
217             RECOGNITION_MODE_USER_IDENTIFICATION,
218     })
219     public @interface RecognitionModes {}
220 
221     /**
222      * Simple recognition of the key phrase.
223      * Returned by {@link #getSupportedRecognitionModes()}
224      */
225     public static final int RECOGNITION_MODE_VOICE_TRIGGER
226             = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER;
227 
228     /**
229      * User identification performed with the keyphrase recognition.
230      * Returned by {@link #getSupportedRecognitionModes()}
231      */
232     public static final int RECOGNITION_MODE_USER_IDENTIFICATION
233             = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION;
234 
235     //-- Audio capabilities. Values in returned bit field for getSupportedAudioCapabilities() --//
236 
237     /** @hide */
238     @Retention(RetentionPolicy.SOURCE)
239     @IntDef(flag = true, prefix = { "AUDIO_CAPABILITY_" }, value = {
240             AUDIO_CAPABILITY_ECHO_CANCELLATION,
241             AUDIO_CAPABILITY_NOISE_SUPPRESSION,
242     })
243     public @interface AudioCapabilities {}
244 
245     /**
246      * If set the underlying module supports AEC.
247      * Returned by {@link #getSupportedAudioCapabilities()}
248      */
249     public static final int AUDIO_CAPABILITY_ECHO_CANCELLATION =
250             SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_ECHO_CANCELLATION;
251 
252     /**
253      * If set, the underlying module supports noise suppression.
254      * Returned by {@link #getSupportedAudioCapabilities()}
255      */
256     public static final int AUDIO_CAPABILITY_NOISE_SUPPRESSION =
257             SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_NOISE_SUPPRESSION;
258 
259     /** @hide */
260     @Retention(RetentionPolicy.SOURCE)
261     @IntDef(flag = true, prefix = { "MODEL_PARAM_" }, value = {
262             MODEL_PARAM_THRESHOLD_FACTOR,
263     })
264     public @interface ModelParams {}
265 
266     /**
267      * Gates returning {@code IllegalStateException} in {@link #initialize(
268      * PersistableBundle, SharedMemory, SoundTrigger.ModuleProperties)} when no DSP module
269      * is available. If the change is not enabled, the existing behavior of not throwing an
270      * exception and delivering {@link STATE_HARDWARE_UNAVAILABLE} is retained.
271      */
272     @ChangeId
273     @EnabledSince(targetSdkVersion = Build.VERSION_CODES.UPSIDE_DOWN_CAKE)
274     static final long THROW_ON_INITIALIZE_IF_NO_DSP = 269165460L;
275 
276     /**
277      * Gates returning {@link Callback#onFailure} and {@link Callback#onUnknownFailure}
278      * when asynchronous exceptions are propagated to the client. If the change is not enabled,
279      * the existing behavior of delivering {@link #STATE_ERROR} is retained.
280      */
281     @ChangeId
282     @EnabledSince(targetSdkVersion = Build.VERSION_CODES.UPSIDE_DOWN_CAKE)
283     static final long SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS = 280471513L;
284 
285     /**
286      * Controls the sensitivity threshold adjustment factor for a given model.
287      * Negative value corresponds to less sensitive model (high threshold) and
288      * a positive value corresponds to a more sensitive model (low threshold).
289      * Default value is 0.
290      */
291     public static final int MODEL_PARAM_THRESHOLD_FACTOR =
292             android.hardware.soundtrigger.ModelParams.THRESHOLD_FACTOR;
293 
294     static final String TAG = "AlwaysOnHotwordDetector";
295     static final boolean DBG = false;
296 
297     private static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR;
298     private static final int STATUS_OK = SoundTrigger.STATUS_OK;
299 
300     private static final int MSG_AVAILABILITY_CHANGED = 1;
301     private static final int MSG_HOTWORD_DETECTED = 2;
302     private static final int MSG_DETECTION_ERROR = 3;
303     private static final int MSG_DETECTION_PAUSE = 4;
304     private static final int MSG_DETECTION_RESUME = 5;
305     private static final int MSG_HOTWORD_REJECTED = 6;
306     private static final int MSG_HOTWORD_STATUS_REPORTED = 7;
307     private static final int MSG_PROCESS_RESTARTED = 8;
308     private static final int MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE = 9;
309     private static final int MSG_DETECTION_SOUND_TRIGGER_FAILURE = 10;
310     private static final int MSG_DETECTION_UNKNOWN_FAILURE = 11;
311 
312     private final String mText;
313     private final Locale mLocale;
314     /**
315      * The metadata of the Keyphrase, derived from the enrollment application.
316      * This may be null if this keyphrase isn't supported by the enrollment application.
317      */
318     @GuardedBy("mLock")
319     @Nullable
320     private KeyphraseMetadata mKeyphraseMetadata;
321     private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
322     private final IVoiceInteractionManagerService mModelManagementService;
323     private IVoiceInteractionSoundTriggerSession mSoundTriggerSession;
324     private final SoundTriggerListener mInternalCallback;
325     private final Callback mExternalCallback;
326     private final Executor mExternalExecutor;
327     private final Handler mHandler;
328     private final IBinder mBinder = new Binder();
329     private final boolean mSupportSandboxedDetectionService;
330     private final String mAttributionTag;
331 
332     @GuardedBy("mLock")
333     private boolean mIsAvailabilityOverriddenByTestApi = false;
334     @GuardedBy("mLock")
335     private int mAvailability = STATE_NOT_READY;
336 
337     /**
338      *  A ModelParamRange is a representation of supported parameter range for a
339      *  given loaded model.
340      */
341     public static final class ModelParamRange {
342         private final SoundTrigger.ModelParamRange mModelParamRange;
343 
344         /** @hide */
ModelParamRange(SoundTrigger.ModelParamRange modelParamRange)345         ModelParamRange(SoundTrigger.ModelParamRange modelParamRange) {
346             mModelParamRange = modelParamRange;
347         }
348 
349         /**
350          * Get the beginning of the param range
351          *
352          * @return The inclusive start of the supported range.
353          */
getStart()354         public int getStart() {
355             return mModelParamRange.getStart();
356         }
357 
358         /**
359          * Get the end of the param range
360          *
361          * @return The inclusive end of the supported range.
362          */
getEnd()363         public int getEnd() {
364             return mModelParamRange.getEnd();
365         }
366 
367         @Override
368         @NonNull
toString()369         public String toString() {
370             return mModelParamRange.toString();
371         }
372 
373         @Override
equals(@ullable Object obj)374         public boolean equals(@Nullable Object obj) {
375             return mModelParamRange.equals(obj);
376         }
377 
378         @Override
hashCode()379         public int hashCode() {
380             return mModelParamRange.hashCode();
381         }
382     }
383 
384     /**
385      * Additional payload for {@link Callback#onDetected}.
386      */
387     public static class EventPayload {
388 
389         /**
390          * Flags for describing the data format provided in the event payload.
391          *
392          * @hide
393          */
394         @Retention(RetentionPolicy.SOURCE)
395         @IntDef(prefix = {"DATA_FORMAT_"}, value = {
396                 DATA_FORMAT_RAW,
397                 DATA_FORMAT_TRIGGER_AUDIO,
398         })
399         public @interface DataFormat {
400         }
401 
402         /**
403          * Data format is not strictly defined by the framework, and the
404          * {@link android.hardware.soundtrigger.SoundTriggerModule} voice engine may populate this
405          * field in any format.
406          */
407         public static final int DATA_FORMAT_RAW = 0;
408 
409         /**
410          * Data format is defined as trigger audio.
411          *
412          * <p>When this format is used, {@link #getCaptureAudioFormat()} can be used to understand
413          * further the audio format for reading the data.
414          *
415          * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
416          */
417         public static final int DATA_FORMAT_TRIGGER_AUDIO = 1;
418 
419         @DataFormat
420         private final int mDataFormat;
421         // Indicates if {@code captureSession} can be used to continue capturing more audio
422         // from the DSP hardware.
423         private final boolean mCaptureAvailable;
424         // The session to use when attempting to capture more audio from the DSP hardware.
425         private final int mCaptureSession;
426         private final AudioFormat mAudioFormat;
427         // Raw data associated with the event.
428         // This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true.
429         private final byte[] mData;
430         private final HotwordDetectedResult mHotwordDetectedResult;
431         private final ParcelFileDescriptor mAudioStream;
432         private final List<KeyphraseRecognitionExtra> mKephraseExtras;
433 
434         @ElapsedRealtimeLong
435         private final long mHalEventReceivedMillis;
436 
437         private final boolean mIsRecognitionStopped;
438 
EventPayload( boolean captureAvailable, @Nullable AudioFormat audioFormat, int captureSession, @DataFormat int dataFormat, @Nullable byte[] data, @Nullable HotwordDetectedResult hotwordDetectedResult, @Nullable ParcelFileDescriptor audioStream, @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras, @ElapsedRealtimeLong long halEventReceivedMillis, boolean isRecognitionStopped)439         private EventPayload(
440                 boolean captureAvailable,
441                 @Nullable AudioFormat audioFormat,
442                 int captureSession,
443                 @DataFormat int dataFormat,
444                 @Nullable byte[] data,
445                 @Nullable HotwordDetectedResult hotwordDetectedResult,
446                 @Nullable ParcelFileDescriptor audioStream,
447                 @NonNull List<KeyphraseRecognitionExtra> keyphraseExtras,
448                 @ElapsedRealtimeLong long halEventReceivedMillis,
449                 boolean isRecognitionStopped) {
450             mCaptureAvailable = captureAvailable;
451             mCaptureSession = captureSession;
452             mAudioFormat = audioFormat;
453             mDataFormat = dataFormat;
454             mData = data;
455             mHotwordDetectedResult = hotwordDetectedResult;
456             mAudioStream = audioStream;
457             mKephraseExtras = keyphraseExtras;
458             mHalEventReceivedMillis = halEventReceivedMillis;
459             mIsRecognitionStopped = isRecognitionStopped;
460         }
461 
462         /**
463          * Gets the format of the audio obtained using {@link #getTriggerAudio()}.
464          * May be null if there's no audio present.
465          */
466         @Nullable
getCaptureAudioFormat()467         public AudioFormat getCaptureAudioFormat() {
468             return mAudioFormat;
469         }
470 
471         /**
472          * Gets the raw audio that triggered the keyphrase.
473          * This may be null if the trigger audio isn't available.
474          * If non-null, the format of the audio can be obtained by calling
475          * {@link #getCaptureAudioFormat()}.
476          *
477          * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
478          * @deprecated Use {@link #getData()} instead.
479          */
480         @Deprecated
481         @Nullable
getTriggerAudio()482         public byte[] getTriggerAudio() {
483             if (mDataFormat == DATA_FORMAT_TRIGGER_AUDIO) {
484                 return mData;
485             } else {
486                 return null;
487             }
488         }
489 
490         /**
491          * Conveys the format of the additional data that is triggered with the keyphrase event.
492          *
493          * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
494          * @see DataFormat
495          */
496         @DataFormat
getDataFormat()497         public int getDataFormat() {
498             return mDataFormat;
499         }
500 
501         /**
502          * Gets additional raw data that is triggered with the keyphrase event.
503          *
504          * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this
505          * field with opaque data for use by system applications who know about voice
506          * engine internals. Data may be null if the field is not populated by the
507          * {@link android.hardware.soundtrigger.SoundTriggerModule}.
508          *
509          * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the
510          * entirety of this buffer is expected to be of the format from
511          * {@link #getCaptureAudioFormat()}.
512          *
513          * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
514          */
515         @Nullable
getData()516         public byte[] getData() {
517             return mData;
518         }
519 
520         /**
521          * Gets the session ID to start a capture from the DSP.
522          * This may be null if streaming capture isn't possible.
523          * If non-null, the format of the audio that can be captured can be
524          * obtained using {@link #getCaptureAudioFormat()}.
525          *
526          * TODO: Candidate for Public API when the API to start capture with a session ID
527          * is made public.
528          *
529          * TODO: Add this to {@link #getCaptureAudioFormat()}:
530          * "Gets the format of the audio obtained using {@link #getTriggerAudio()}
531          * or {@link #getCaptureSession()}. May be null if no audio can be obtained
532          * for either the trigger or a streaming session."
533          *
534          * TODO: Should this return a known invalid value instead?
535          *
536          * @hide
537          */
538         @Nullable
539         @UnsupportedAppUsage
getCaptureSession()540         public Integer getCaptureSession() {
541             if (mCaptureAvailable) {
542                 return mCaptureSession;
543             } else {
544                 return null;
545             }
546         }
547 
548         /**
549          * Returns {@link HotwordDetectedResult} associated with the hotword event, passed from
550          * {@link HotwordDetectionService}.
551          */
552         @Nullable
getHotwordDetectedResult()553         public HotwordDetectedResult getHotwordDetectedResult() {
554             return mHotwordDetectedResult;
555         }
556 
557         /**
558          * Returns a stream with bytes corresponding to the open audio stream with hotword data.
559          *
560          * <p>This data represents an audio stream in the format returned by
561          * {@link #getCaptureAudioFormat}.
562          *
563          * <p>Clients are expected to start consuming the stream within 1 second of receiving the
564          * event.
565          *
566          * <p>When this method returns a non-null, clients must close this stream when it's no
567          * longer needed. Failing to do so will result in microphone being open for longer periods
568          * of time, and app being attributed for microphone usage.
569          */
570         @Nullable
getAudioStream()571         public ParcelFileDescriptor getAudioStream() {
572             return mAudioStream;
573         }
574 
575         /**
576          * Returns the keyphrases recognized by the voice engine with additional confidence
577          * information
578          *
579          * @return List of keyphrase extras describing additional data for each keyphrase the voice
580          * engine triggered on for this event. The ordering of the list is preserved based on what
581          * the ordering provided by {@link android.hardware.soundtrigger.SoundTriggerModule}.
582          */
583         @NonNull
getKeyphraseRecognitionExtras()584         public List<KeyphraseRecognitionExtra> getKeyphraseRecognitionExtras() {
585             return mKephraseExtras;
586         }
587 
588         /**
589          * Timestamp of when the trigger event from SoundTriggerHal was received by the framework.
590          *
591          * Clock monotonic including suspend time or its equivalent on the system,
592          * in the same units and timebase as {@link SystemClock#elapsedRealtime()}.
593          *
594          * @return Elapsed realtime in milliseconds when the event was received from the HAL.
595          *      Returns -1 if the event was not generated from the HAL.
596          */
597         @ElapsedRealtimeLong
getHalEventReceivedMillis()598         public long getHalEventReceivedMillis() {
599             return mHalEventReceivedMillis;
600         }
601 
602         /** Returns whether the system has stopped hotword recognition because of this detection. */
603         @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)
isRecognitionStopped()604         public boolean isRecognitionStopped() {
605             return mIsRecognitionStopped;
606         }
607 
608         /**
609          * Builder class for {@link EventPayload} objects
610          *
611          * @hide
612          */
613         @TestApi
614         public static final class Builder {
615             private boolean mCaptureAvailable = false;
616             private int mCaptureSession = -1;
617             private AudioFormat mAudioFormat = null;
618             @DataFormat
619             private int mDataFormat = DATA_FORMAT_RAW;
620             private byte[] mData = null;
621             private HotwordDetectedResult mHotwordDetectedResult = null;
622             private ParcelFileDescriptor mAudioStream = null;
623             private List<KeyphraseRecognitionExtra> mKeyphraseExtras = Collections.emptyList();
624             @ElapsedRealtimeLong
625             private long mHalEventReceivedMillis = -1;
626             // default to true to keep prior behavior
627             private boolean mIsRecognitionStopped = true;
628 
Builder()629             public Builder() {}
630 
Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent)631             Builder(SoundTrigger.KeyphraseRecognitionEvent keyphraseRecognitionEvent) {
632                 setCaptureAvailable(keyphraseRecognitionEvent.isCaptureAvailable());
633                 setCaptureSession(keyphraseRecognitionEvent.getCaptureSession());
634                 if (keyphraseRecognitionEvent.getCaptureFormat() != null) {
635                     setCaptureAudioFormat(keyphraseRecognitionEvent.getCaptureFormat());
636                 }
637                 setDataFormat((keyphraseRecognitionEvent.triggerInData) ? DATA_FORMAT_TRIGGER_AUDIO
638                         : DATA_FORMAT_RAW);
639                 if (keyphraseRecognitionEvent.getData() != null) {
640                     setData(keyphraseRecognitionEvent.getData());
641                 }
642                 if (keyphraseRecognitionEvent.keyphraseExtras != null) {
643                     setKeyphraseRecognitionExtras(
644                             Arrays.asList(keyphraseRecognitionEvent.keyphraseExtras));
645                 }
646                 setHalEventReceivedMillis(keyphraseRecognitionEvent.getHalEventReceivedMillis());
647             }
648 
649             /**
650              * Indicates if {@code captureSession} can be used to continue capturing more audio from
651              * the DSP hardware.
652              */
653             @SuppressLint("MissingGetterMatchingBuilder")
654             @NonNull
setCaptureAvailable(boolean captureAvailable)655             public Builder setCaptureAvailable(boolean captureAvailable) {
656                 mCaptureAvailable = captureAvailable;
657                 return this;
658             }
659 
660             /**
661              * Sets the session ID to start a capture from the DSP.
662              */
663             @SuppressLint("MissingGetterMatchingBuilder")
664             @NonNull
setCaptureSession(int captureSession)665             public Builder setCaptureSession(int captureSession) {
666                 mCaptureSession = captureSession;
667                 return this;
668             }
669 
670             /**
671              * Sets the format of the audio obtained using {@link #getTriggerAudio()}.
672              */
673             @NonNull
setCaptureAudioFormat(@onNull AudioFormat audioFormat)674             public Builder setCaptureAudioFormat(@NonNull AudioFormat audioFormat) {
675                 mAudioFormat = audioFormat;
676                 return this;
677             }
678 
679             /**
680              * Conveys the format of the additional data that is triggered with the keyphrase event.
681              *
682              * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
683              * @see DataFormat
684              */
685             @NonNull
setDataFormat(@ataFormat int dataFormat)686             public Builder setDataFormat(@DataFormat int dataFormat) {
687                 mDataFormat = dataFormat;
688                 return this;
689             }
690 
691             /**
692              * Sets additional raw data that is triggered with the keyphrase event.
693              *
694              * <p>A {@link android.hardware.soundtrigger.SoundTriggerModule} may populate this
695              * field with opaque data for use by system applications who know about voice
696              * engine internals. Data may be null if the field is not populated by the
697              * {@link android.hardware.soundtrigger.SoundTriggerModule}.
698              *
699              * <p>If {@link #getDataFormat()} is {@link #DATA_FORMAT_TRIGGER_AUDIO}, then the
700              * entirety of this
701              * buffer is expected to be of the format from {@link #getCaptureAudioFormat()}.
702              *
703              * @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
704              */
705             @NonNull
setData(@onNull byte[] data)706             public Builder setData(@NonNull byte[] data) {
707                 mData = data;
708                 return this;
709             }
710 
711             /**
712              * Sets {@link HotwordDetectedResult} associated with the hotword event, passed from
713              * {@link HotwordDetectionService}.
714              */
715             @NonNull
setHotwordDetectedResult( @onNull HotwordDetectedResult hotwordDetectedResult)716             public Builder setHotwordDetectedResult(
717                     @NonNull HotwordDetectedResult hotwordDetectedResult) {
718                 mHotwordDetectedResult = hotwordDetectedResult;
719                 return this;
720             }
721 
722             /**
723              * Sets a stream with bytes corresponding to the open audio stream with hotword data.
724              *
725              * <p>This data represents an audio stream in the format returned by
726              * {@link #getCaptureAudioFormat}.
727              *
728              * <p>Clients are expected to start consuming the stream within 1 second of receiving
729              * the
730              * event.
731              */
732             @NonNull
setAudioStream(@onNull ParcelFileDescriptor audioStream)733             public Builder setAudioStream(@NonNull ParcelFileDescriptor audioStream) {
734                 mAudioStream = audioStream;
735                 return this;
736             }
737 
738             /**
739              * Sets the keyphrases recognized by the voice engine with additional confidence
740              * information
741              */
742             @NonNull
setKeyphraseRecognitionExtras( @onNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)743             public Builder setKeyphraseRecognitionExtras(
744                     @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) {
745                 mKeyphraseExtras = keyphraseRecognitionExtras;
746                 return this;
747             }
748 
749             /**
750              * Timestamp of when the trigger event from SoundTriggerHal was received by the
751              * framework.
752              *
753              * Clock monotonic including suspend time or its equivalent on the system,
754              * in the same units and timebase as {@link SystemClock#elapsedRealtime()}.
755              */
756             @NonNull
setHalEventReceivedMillis( @lapsedRealtimeLong long halEventReceivedMillis)757             public Builder setHalEventReceivedMillis(
758                     @ElapsedRealtimeLong long halEventReceivedMillis) {
759                 mHalEventReceivedMillis = halEventReceivedMillis;
760                 return this;
761             }
762 
763             /**
764              * Sets whether the system has stopped hotword recognition because of this detection.
765              */
766             @FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)
767             @NonNull
setIsRecognitionStopped(boolean isRecognitionStopped)768             public Builder setIsRecognitionStopped(boolean isRecognitionStopped) {
769                 mIsRecognitionStopped = isRecognitionStopped;
770                 return this;
771             }
772 
773             /**
774              * Builds an {@link EventPayload} instance
775              */
776             @NonNull
build()777             public EventPayload build() {
778                 return new EventPayload(
779                         mCaptureAvailable,
780                         mAudioFormat,
781                         mCaptureSession,
782                         mDataFormat,
783                         mData,
784                         mHotwordDetectedResult,
785                         mAudioStream,
786                         mKeyphraseExtras,
787                         mHalEventReceivedMillis,
788                         mIsRecognitionStopped);
789             }
790         }
791     }
792 
793     /**
794      * Callbacks for always-on hotword detection.
795      */
796     public abstract static class Callback implements HotwordDetector.Callback {
797 
798         /**
799          * Updates the availability state of the active keyphrase and locale on every keyphrase
800          * sound model change.
801          *
802          * <p>This API is called whenever there's a possibility that the keyphrase associated
803          * with this detector has been updated. It is not guaranteed that there is in fact any
804          * change, as it may be called for other reasons.</p>
805          *
806          * <p>This API is also guaranteed to be called right after an AlwaysOnHotwordDetector
807          * instance is created to updated the current availability state.</p>
808          *
809          * <p>Availability implies the current enrollment state of the given keyphrase. If the
810          * hardware on this system is not capable of listening for the given keyphrase,
811          * {@link AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE} will be returned.
812          *
813          * @see AlwaysOnHotwordDetector#STATE_HARDWARE_UNAVAILABLE
814          * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_UNENROLLED
815          * @see AlwaysOnHotwordDetector#STATE_KEYPHRASE_ENROLLED
816          * @see AlwaysOnHotwordDetector#STATE_ERROR
817          */
onAvailabilityChanged(int status)818         public abstract void onAvailabilityChanged(int status);
819 
820         /**
821          * Called when the keyphrase is spoken.
822          *
823          * <p>If {@code eventPayload.isRecognitionStopped()} returns true, this implicitly stops
824          * listening for the keyphrase once it's detected. Clients should start a recognition again
825          * once they are done handling this detection.
826          *
827          * @param eventPayload Payload data for the detection event. This may contain the trigger
828          *     audio, if requested when calling {@link
829          *     AlwaysOnHotwordDetector#startRecognition(int)} or if the audio comes from the {@link
830          *     android.service.wearable.WearableSensingService}.
831          */
onDetected(@onNull EventPayload eventPayload)832         public abstract void onDetected(@NonNull EventPayload eventPayload);
833 
834         /**
835          * {@inheritDoc}
836          *
837          * @deprecated On {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE} and above,
838          * implement {@link HotwordDetector.Callback#onFailure(HotwordDetectionServiceFailure)},
839          * {@link AlwaysOnHotwordDetector.Callback#onFailure(SoundTriggerFailure)},
840          * {@link HotwordDetector.Callback#onUnknownFailure(String)} instead.
841          */
842         @Deprecated
843         @Override
onError()844         public abstract void onError();
845 
846         /**
847          * Called when the detection fails due to an error occurs in the
848          * {@link com.android.server.soundtrigger.SoundTriggerService} and
849          * {@link com.android.server.soundtrigger_middleware.SoundTriggerMiddlewareService},
850          * {@link SoundTriggerFailure} will be reported to the detector.
851          *
852          * @param soundTriggerFailure It provides the error code, error message and suggested
853          *                            action.
854          */
onFailure(@onNull SoundTriggerFailure soundTriggerFailure)855         public void onFailure(@NonNull SoundTriggerFailure soundTriggerFailure) {
856             onError();
857         }
858 
859         /** {@inheritDoc} */
onRecognitionPaused()860         public abstract void onRecognitionPaused();
861 
862         /** {@inheritDoc} */
onRecognitionResumed()863         public abstract void onRecognitionResumed();
864 
865         /** {@inheritDoc} */
onRejected(@onNull HotwordRejectedResult result)866         public void onRejected(@NonNull HotwordRejectedResult result) {
867         }
868 
869         /** {@inheritDoc} */
onHotwordDetectionServiceInitialized(int status)870         public void onHotwordDetectionServiceInitialized(int status) {
871         }
872 
873         /** {@inheritDoc} */
onHotwordDetectionServiceRestarted()874         public void onHotwordDetectionServiceRestarted() {
875         }
876     }
877 
878     /**
879      * @param text The keyphrase text to get the detector for.
880      * @param locale The java locale for the detector.
881      * @param callback A non-null Callback for receiving the recognition events.
882      * @param modelManagementService A service that allows management of sound models.
883      * @param targetSdkVersion The target SDK version.
884      * @param SupportSandboxedDetectionService {@code true} if HotwordDetectionService should be
885      * triggered, otherwise {@code false}.
886      * @param attributionTag an optional attribution tag passed form the
887      * {@link VoiceInteractionService} context via the
888      * {@link createAlwaysOnHotwordDetectorInternal(String, Locale, boolean, PersistableBundle,
889      * SharedMemory, ModuleProperties, Executor, Callback)}.
890      *
891      * @hide
892      */
AlwaysOnHotwordDetector(String text, Locale locale, Executor executor, Callback callback, KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, IVoiceInteractionManagerService modelManagementService, int targetSdkVersion, boolean supportSandboxedDetectionService, @Nullable String attributionTag)893     public AlwaysOnHotwordDetector(String text, Locale locale, Executor executor, Callback callback,
894             KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
895             IVoiceInteractionManagerService modelManagementService, int targetSdkVersion,
896             boolean supportSandboxedDetectionService, @Nullable String attributionTag) {
897         super(modelManagementService, executor, callback);
898 
899         mHandler = new MyHandler(Looper.getMainLooper());
900         mText = text;
901         mLocale = locale;
902         mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
903         mExternalCallback = callback;
904         mExternalExecutor = executor != null ? executor : new HandlerExecutor(
905                 new Handler(Looper.myLooper()));
906         mInternalCallback = new SoundTriggerListener(mHandler);
907         mModelManagementService = modelManagementService;
908         mSupportSandboxedDetectionService = supportSandboxedDetectionService;
909         mAttributionTag = attributionTag;
910     }
911 
912     // Do nothing. This method should not be abstract.
913     // TODO (b/269355519) un-subclass AOHD.
914     @Override
initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)915     void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) {}
916 
initialize(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, @Nullable SoundTrigger.ModuleProperties moduleProperties)917     void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory,
918             @Nullable SoundTrigger.ModuleProperties moduleProperties) {
919         if (mSupportSandboxedDetectionService) {
920             initAndVerifyDetector(options, sharedMemory, mInternalCallback,
921                     DETECTOR_TYPE_TRUSTED_HOTWORD_DSP, mAttributionTag);
922         }
923         try {
924             Identity identity = new Identity();
925             identity.packageName = ActivityThread.currentOpPackageName();
926             if (IS_IDENTITY_WITH_ATTRIBUTION_TAG) {
927                 identity.attributionTag = mAttributionTag;
928             }
929             if (moduleProperties == null) {
930                 moduleProperties = mModelManagementService
931                         .listModuleProperties(identity)
932                         .stream()
933                         .filter(prop -> !prop.getSupportedModelArch()
934                                 .equals(SoundTrigger.FAKE_HAL_ARCH))
935                         .findFirst()
936                         .orElse(null);
937                 if (CompatChanges.isChangeEnabled(THROW_ON_INITIALIZE_IF_NO_DSP) &&
938                         moduleProperties == null) {
939                     throw new IllegalStateException("No DSP module available to attach to");
940                 }
941             }
942             mSoundTriggerSession =
943                     mModelManagementService.createSoundTriggerSessionAsOriginator(
944                             identity, mBinder, moduleProperties);
945         } catch (RemoteException e) {
946             throw e.rethrowAsRuntimeException();
947         }
948         new RefreshAvailabilityTask().execute();
949     }
950 
951     /**
952      * {@inheritDoc}
953      *
954      * @throws IllegalStateException if this AlwaysOnHotwordDetector wasn't specified to use a
955      * {@link HotwordDetectionService} when it was created. In addition, if this
956      * AlwaysOnHotwordDetector is in an invalid or error state.
957      */
958     @Override
updateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory)959     public final void updateState(@Nullable PersistableBundle options,
960             @Nullable SharedMemory sharedMemory) {
961         synchronized (mLock) {
962             if (!mSupportSandboxedDetectionService) {
963                 throw new IllegalStateException(
964                         "updateState called, but it doesn't support hotword detection service");
965             }
966             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
967                 throw new IllegalStateException(
968                         "updateState called on an invalid detector or error state");
969             }
970         }
971 
972         super.updateState(options, sharedMemory);
973     }
974 
975     /**
976      * Test API for manipulating the voice engine and sound model availability.
977      *
978      * After overriding the availability status, the client's
979      * {@link Callback#onAvailabilityChanged(int)} will be called to reflect the updated state.
980      *
981      * When this override is set, all system updates to availability will be ignored.
982      * @hide
983      */
984     @TestApi
overrideAvailability(int availability)985     public void overrideAvailability(int availability) {
986         synchronized (mLock) {
987             mAvailability = availability;
988             mIsAvailabilityOverriddenByTestApi = true;
989             // ENROLLED state requires there to be metadata about the sound model so a fake one
990             // is created.
991             if (mKeyphraseMetadata == null && mAvailability == STATE_KEYPHRASE_ENROLLED) {
992                 Set<Locale> fakeSupportedLocales = new HashSet<>();
993                 fakeSupportedLocales.add(mLocale);
994                 mKeyphraseMetadata = new KeyphraseMetadata(1, mText, fakeSupportedLocales,
995                         AlwaysOnHotwordDetector.RECOGNITION_MODE_VOICE_TRIGGER);
996             }
997             notifyStateChangedLocked();
998         }
999     }
1000 
1001     /**
1002      * Test API for clearing an availability override set by {@link #overrideAvailability(int)}
1003      *
1004      * This method will restore the availability to the current system state.
1005      * @hide
1006      */
1007     @TestApi
resetAvailability()1008     public void resetAvailability() {
1009         synchronized (mLock) {
1010             mIsAvailabilityOverriddenByTestApi = false;
1011         }
1012         // Execute a refresh availability task - which should then notify of a change.
1013         new RefreshAvailabilityTask().execute();
1014     }
1015 
1016     /**
1017      * Test API to simulate to trigger hardware recognition event for test.
1018      *
1019      * @hide
1020      */
1021     @TestApi
1022     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
triggerHardwareRecognitionEventForTest(int status, int soundModelHandle, @ElapsedRealtimeLong long halEventReceivedMillis, boolean captureAvailable, int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData, @NonNull AudioFormat captureFormat, @Nullable byte[] data, @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras)1023     public void triggerHardwareRecognitionEventForTest(int status, int soundModelHandle,
1024             @ElapsedRealtimeLong long halEventReceivedMillis, boolean captureAvailable,
1025             int captureSession, int captureDelayMs, int capturePreambleMs, boolean triggerInData,
1026             @NonNull AudioFormat captureFormat, @Nullable byte[] data,
1027             @NonNull List<KeyphraseRecognitionExtra> keyphraseRecognitionExtras) {
1028         Log.d(TAG, "triggerHardwareRecognitionEventForTest()");
1029         synchronized (mLock) {
1030             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1031                 throw new IllegalStateException("triggerHardwareRecognitionEventForTest called on"
1032                         + " an invalid detector or error state");
1033             }
1034             try {
1035                 mModelManagementService.triggerHardwareRecognitionEventForTest(
1036                         new KeyphraseRecognitionEvent(status, soundModelHandle, captureAvailable,
1037                                 captureSession, captureDelayMs, capturePreambleMs, triggerInData,
1038                                 captureFormat, data, keyphraseRecognitionExtras.toArray(
1039                                 new KeyphraseRecognitionExtra[0]), halEventReceivedMillis,
1040                                 new Binder()),
1041                         mInternalCallback);
1042             } catch (RemoteException e) {
1043                 throw e.rethrowFromSystemServer();
1044             }
1045         }
1046     }
1047 
1048     /**
1049      * Gets the recognition modes supported by the associated keyphrase.
1050      *
1051      * @see #RECOGNITION_MODE_USER_IDENTIFICATION
1052      * @see #RECOGNITION_MODE_VOICE_TRIGGER
1053      *
1054      * @throws UnsupportedOperationException if the keyphrase itself isn't supported.
1055      *         Callers should only call this method after a supported state callback on
1056      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1057      * @throws IllegalStateException if the detector is in an invalid or error state.
1058      *         This may happen if another detector has been instantiated or the
1059      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1060      */
getSupportedRecognitionModes()1061     public @RecognitionModes int getSupportedRecognitionModes() {
1062         if (DBG) Slog.d(TAG, "getSupportedRecognitionModes()");
1063         synchronized (mLock) {
1064             return getSupportedRecognitionModesLocked();
1065         }
1066     }
1067 
1068     @GuardedBy("mLock")
getSupportedRecognitionModesLocked()1069     private int getSupportedRecognitionModesLocked() {
1070         if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1071             throw new IllegalStateException(
1072                     "getSupportedRecognitionModes called on an invalid detector or error state");
1073         }
1074 
1075         // This method only makes sense if we can actually support a recognition.
1076         if (mAvailability != STATE_KEYPHRASE_ENROLLED || mKeyphraseMetadata == null) {
1077             throw new UnsupportedOperationException(
1078                     "Getting supported recognition modes for the keyphrase is not supported");
1079         }
1080 
1081         return mKeyphraseMetadata.getRecognitionModeFlags();
1082     }
1083 
1084     /**
1085      * Get the audio capabilities supported by the platform which can be enabled when
1086      * starting a recognition.
1087      * Caller must be the active voice interaction service via
1088      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1089      *
1090      * @see #AUDIO_CAPABILITY_ECHO_CANCELLATION
1091      * @see #AUDIO_CAPABILITY_NOISE_SUPPRESSION
1092      *
1093      * @return Bit field encoding of the AudioCapabilities supported.
1094      */
1095     @AudioCapabilities
getSupportedAudioCapabilities()1096     public int getSupportedAudioCapabilities() {
1097         if (DBG) Slog.d(TAG, "getSupportedAudioCapabilities()");
1098         synchronized (mLock) {
1099             return getSupportedAudioCapabilitiesLocked();
1100         }
1101     }
1102 
1103     @GuardedBy("mLock")
getSupportedAudioCapabilitiesLocked()1104     private int getSupportedAudioCapabilitiesLocked() {
1105         try {
1106             ModuleProperties properties =
1107                     mSoundTriggerSession.getDspModuleProperties();
1108             if (properties != null) {
1109                 return properties.getAudioCapabilities();
1110             }
1111 
1112             return 0;
1113         } catch (RemoteException e) {
1114             throw e.rethrowFromSystemServer();
1115         }
1116     }
1117 
1118     /**
1119      * Starts recognition for the associated keyphrase.
1120      * Caller must be the active voice interaction service via
1121      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1122      *
1123      * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
1124      * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS
1125      *
1126      * @param recognitionFlags The flags to control the recognition properties.
1127      * @param data Additional pass-through data to the system voice engine along with the
1128      *             startRecognition request. This data is intended to provide additional parameters
1129      *             when starting the opaque sound model.
1130      * @return Indicates whether the call succeeded or not.
1131      * @throws UnsupportedOperationException if the recognition isn't supported.
1132      *         Callers should only call this method after a supported state callback on
1133      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1134      * @throws IllegalStateException if the detector is in an invalid or error state.
1135      *         This may happen if another detector has been instantiated or the
1136      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1137      */
1138     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
startRecognition(@ecognitionFlags int recognitionFlags, @NonNull byte[] data)1139     public boolean startRecognition(@RecognitionFlags int recognitionFlags, @NonNull byte[] data) {
1140         synchronized (mLock) {
1141             return startRecognitionLocked(recognitionFlags, data)
1142                     == STATUS_OK;
1143         }
1144     }
1145 
1146     /**
1147      * Starts recognition for the associated keyphrase.
1148      * Caller must be the active voice interaction service via
1149      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1150      *
1151      * @see #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
1152      * @see #RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS
1153      *
1154      * @param recognitionFlags The flags to control the recognition properties.
1155      * @return Indicates whether the call succeeded or not.
1156      * @throws UnsupportedOperationException if the recognition isn't supported.
1157      *         Callers should only call this method after a supported state callback on
1158      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1159      * @throws IllegalStateException if the detector is in an invalid or error state.
1160      *         This may happen if another detector has been instantiated or the
1161      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1162      */
1163     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
startRecognition(@ecognitionFlags int recognitionFlags)1164     public boolean startRecognition(@RecognitionFlags int recognitionFlags) {
1165         if (DBG) Slog.d(TAG, "startRecognition(" + recognitionFlags + ")");
1166         synchronized (mLock) {
1167             return startRecognitionLocked(recognitionFlags, /* data= */new byte[0]) == STATUS_OK;
1168         }
1169     }
1170 
1171     /**
1172      * Starts recognition for the associated keyphrase.
1173      *
1174      * @see #startRecognition(int)
1175      */
1176     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
1177     @Override
startRecognition()1178     public boolean startRecognition() {
1179         return startRecognition(0);
1180     }
1181 
1182     /**
1183      * Stops recognition for the associated keyphrase.
1184      * Caller must be the active voice interaction service via
1185      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1186      *
1187      * @return Indicates whether the call succeeded or not.
1188      * @throws UnsupportedOperationException if the recognition isn't supported.
1189      *         Callers should only call this method after a supported state callback on
1190      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1191      * @throws IllegalStateException if the detector is in an invalid or error state.
1192      *         This may happen if another detector has been instantiated or the
1193      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1194      */
1195     // TODO: Remove this RequiresPermission since it isn't actually enforced. Also fix the javadoc
1196     // about permissions enforcement (when it throws vs when it just returns false) for other
1197     // methods in this class.
1198     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
1199     @Override
stopRecognition()1200     public boolean stopRecognition() {
1201         if (DBG) Slog.d(TAG, "stopRecognition()");
1202         synchronized (mLock) {
1203             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1204                 throw new IllegalStateException(
1205                         "stopRecognition called on an invalid detector or error state");
1206             }
1207 
1208             // Check if we can start/stop a recognition.
1209             if (mAvailability != STATE_KEYPHRASE_ENROLLED) {
1210                 throw new UnsupportedOperationException(
1211                         "Recognition for the given keyphrase is not supported");
1212             }
1213 
1214             return stopRecognitionLocked() == STATUS_OK;
1215         }
1216     }
1217 
1218     /**
1219      * Set a model specific {@link ModelParams} with the given value. This
1220      * parameter will keep its value for the duration the model is loaded regardless of starting and
1221      * stopping recognition. Once the model is unloaded, the value will be lost.
1222      * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before calling this
1223      * method.
1224      * Caller must be the active voice interaction service via
1225      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1226      *
1227      * @param modelParam   {@link ModelParams}
1228      * @param value        Value to set
1229      * @return - {@link SoundTrigger#STATUS_OK} in case of success
1230      *         - {@link SoundTrigger#STATUS_NO_INIT} if the native service cannot be reached
1231      *         - {@link SoundTrigger#STATUS_BAD_VALUE} invalid input parameter
1232      *         - {@link SoundTrigger#STATUS_INVALID_OPERATION} if the call is out of sequence or
1233      *           if API is not supported by HAL
1234      * @throws IllegalStateException if the detector is in an invalid or error state.
1235      *         This may happen if another detector has been instantiated or the
1236      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1237      */
1238     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
setParameter(@odelParams int modelParam, int value)1239     public int setParameter(@ModelParams int modelParam, int value) {
1240         if (DBG) {
1241             Slog.d(TAG, "setParameter(" + modelParam + ", " + value + ")");
1242         }
1243 
1244         synchronized (mLock) {
1245             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1246                 throw new IllegalStateException(
1247                         "setParameter called on an invalid detector or error state");
1248             }
1249 
1250             return setParameterLocked(modelParam, value);
1251         }
1252     }
1253 
1254     /**
1255      * Get a model specific {@link ModelParams}. This parameter will keep its value
1256      * for the duration the model is loaded regardless of starting and stopping recognition.
1257      * Once the model is unloaded, the value will be lost. If the value is not set, a default
1258      * value is returned. See {@link ModelParams} for parameter default values.
1259      * {@link AlwaysOnHotwordDetector#queryParameter} should be checked first before
1260      * calling this method.
1261      * Caller must be the active voice interaction service via
1262      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1263      *
1264      * @param modelParam   {@link ModelParams}
1265      * @return value of parameter
1266      * @throws IllegalStateException if the detector is in an invalid or error state.
1267      *         This may happen if another detector has been instantiated or the
1268      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1269      */
1270     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
getParameter(@odelParams int modelParam)1271     public int getParameter(@ModelParams int modelParam) {
1272         if (DBG) {
1273             Slog.d(TAG, "getParameter(" + modelParam + ")");
1274         }
1275 
1276         synchronized (mLock) {
1277             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1278                 throw new IllegalStateException(
1279                         "getParameter called on an invalid detector or error state");
1280             }
1281 
1282             return getParameterLocked(modelParam);
1283         }
1284     }
1285 
1286     /**
1287      * Determine if parameter control is supported for the given model handle.
1288      * This method should be checked prior to calling {@link AlwaysOnHotwordDetector#setParameter}
1289      * or {@link AlwaysOnHotwordDetector#getParameter}.
1290      * Caller must be the active voice interaction service via
1291      * Settings.Secure.VOICE_INTERACTION_SERVICE.
1292      *
1293      * @param modelParam {@link ModelParams}
1294      * @return supported range of parameter, null if not supported
1295      * @throws IllegalStateException if the detector is in an invalid or error state.
1296      *         This may happen if another detector has been instantiated or the
1297      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1298      */
1299     @RequiresPermission(allOf = {RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD})
1300     @Nullable
queryParameter(@odelParams int modelParam)1301     public ModelParamRange queryParameter(@ModelParams int modelParam) {
1302         if (DBG) {
1303             Slog.d(TAG, "queryParameter(" + modelParam + ")");
1304         }
1305 
1306         synchronized (mLock) {
1307             if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1308                 throw new IllegalStateException(
1309                         "queryParameter called on an invalid detector or error state");
1310             }
1311 
1312             return queryParameterLocked(modelParam);
1313         }
1314     }
1315 
1316     /**
1317      * Creates an intent to start the enrollment for the associated keyphrase.
1318      * This intent must be invoked using {@link Context#startForegroundService(Intent)}.
1319      * Starting re-enrollment is only valid if the keyphrase is un-enrolled,
1320      * i.e. {@link #STATE_KEYPHRASE_UNENROLLED},
1321      * otherwise {@link #createReEnrollIntent()} should be preferred.
1322      *
1323      * @return An {@link Intent} to start enrollment for the given keyphrase.
1324      * @throws UnsupportedOperationException if managing they keyphrase isn't supported.
1325      *         Callers should only call this method after a supported state callback on
1326      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1327      * @throws IllegalStateException if the detector is in an invalid state.
1328      *         This may happen if another detector has been instantiated or the
1329      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1330      */
1331     @Nullable
createEnrollIntent()1332     public Intent createEnrollIntent() {
1333         if (DBG) Slog.d(TAG, "createEnrollIntent");
1334         synchronized (mLock) {
1335             return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_ENROLL);
1336         }
1337     }
1338 
1339     /**
1340      * Creates an intent to start the un-enrollment for the associated keyphrase.
1341      * This intent must be invoked using {@link Context#startForegroundService(Intent)}.
1342      * Starting re-enrollment is only valid if the keyphrase is already enrolled,
1343      * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error.
1344      *
1345      * @return An {@link Intent} to start un-enrollment for the given keyphrase.
1346      * @throws UnsupportedOperationException if managing they keyphrase isn't supported.
1347      *         Callers should only call this method after a supported state callback on
1348      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1349      * @throws IllegalStateException if the detector is in an invalid state.
1350      *         This may happen if another detector has been instantiated or the
1351      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1352      */
1353     @Nullable
createUnEnrollIntent()1354     public Intent createUnEnrollIntent() {
1355         if (DBG) Slog.d(TAG, "createUnEnrollIntent");
1356         synchronized (mLock) {
1357             return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_UN_ENROLL);
1358         }
1359     }
1360 
1361     /**
1362      * Creates an intent to start the re-enrollment for the associated keyphrase.
1363      * This intent must be invoked using {@link Context#startForegroundService(Intent)}.
1364      * Starting re-enrollment is only valid if the keyphrase is already enrolled,
1365      * i.e. {@link #STATE_KEYPHRASE_ENROLLED}, otherwise invoking this may result in an error.
1366      *
1367      * @return An {@link Intent} to start re-enrollment for the given keyphrase.
1368      * @throws UnsupportedOperationException if managing they keyphrase isn't supported.
1369      *         Callers should only call this method after a supported state callback on
1370      *         {@link Callback#onAvailabilityChanged(int)} to avoid this exception.
1371      * @throws IllegalStateException if the detector is in an invalid or error state.
1372      *         This may happen if another detector has been instantiated or the
1373      *         {@link VoiceInteractionService} hosting this detector has been shut down.
1374      */
1375     @Nullable
createReEnrollIntent()1376     public Intent createReEnrollIntent() {
1377         if (DBG) Slog.d(TAG, "createReEnrollIntent");
1378         synchronized (mLock) {
1379             return getManageIntentLocked(KeyphraseEnrollmentInfo.MANAGE_ACTION_RE_ENROLL);
1380         }
1381     }
1382 
1383     @GuardedBy("mLock")
getManageIntentLocked(@eyphraseEnrollmentInfo.ManageActions int action)1384     private Intent getManageIntentLocked(@KeyphraseEnrollmentInfo.ManageActions int action) {
1385         if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1386             throw new IllegalStateException(
1387                     "getManageIntent called on an invalid detector or error state");
1388         }
1389 
1390         // This method only makes sense if we can actually support a recognition.
1391         if (mAvailability != STATE_KEYPHRASE_ENROLLED
1392                 && mAvailability != STATE_KEYPHRASE_UNENROLLED) {
1393             throw new UnsupportedOperationException(
1394                     "Managing the given keyphrase is not supported");
1395         }
1396 
1397         return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);
1398     }
1399 
1400     /** {@inheritDoc} */
1401     @Override
destroy()1402     public void destroy() {
1403         synchronized (mLock) {
1404             detachSessionLocked();
1405 
1406             mAvailability = STATE_INVALID;
1407             mIsAvailabilityOverriddenByTestApi = false;
1408             notifyStateChangedLocked();
1409         }
1410         super.destroy();
1411     }
1412 
detachSessionLocked()1413     private void detachSessionLocked() {
1414         try {
1415             if (DBG) Slog.d(TAG, "detachSessionLocked() " + mSoundTriggerSession);
1416             if (mSoundTriggerSession != null) {
1417                 mSoundTriggerSession.detach();
1418             }
1419         } catch (RemoteException e) {
1420             e.rethrowFromSystemServer();
1421         }
1422     }
1423 
1424     /**
1425      * @hide
1426      */
1427     @Override
isUsingSandboxedDetectionService()1428     public boolean isUsingSandboxedDetectionService() {
1429         return mSupportSandboxedDetectionService;
1430     }
1431 
1432     /**
1433      * Reloads the sound models from the service.
1434      *
1435      * @hide
1436      */
1437     // TODO(b/281608561): remove the enrollment flow from AlwaysOnHotwordDetector
onSoundModelsChanged()1438     void onSoundModelsChanged() {
1439         synchronized (mLock) {
1440             if (mAvailability == STATE_INVALID
1441                     || mAvailability == STATE_HARDWARE_UNAVAILABLE
1442                     || mAvailability == STATE_ERROR) {
1443                 Slog.w(TAG, "Received onSoundModelsChanged for an unsupported keyphrase/config"
1444                         + " or in the error state");
1445                 return;
1446             }
1447 
1448             // Because this method reflects an update from the system service models, we should not
1449             // update the client of an availability change when the availability has been overridden
1450             // via a test API.
1451             if (mIsAvailabilityOverriddenByTestApi) {
1452                 Slog.w(TAG, "Suppressing system availability update. "
1453                         + "Availability is overridden by test API.");
1454                 return;
1455             }
1456 
1457             // Stop the recognition before proceeding if we are in the enrolled state.
1458             // The framework makes the guarantee that an actively used model is present in the
1459             // system server's enrollment database. For this reason we much stop an actively running
1460             // model when the underlying sound model in enrollment database no longer match.
1461             if (mAvailability == STATE_KEYPHRASE_ENROLLED) {
1462                 // A SoundTriggerFailure will be sent to the client if the model state was
1463                 // changed. This is an overloading of the onFailure usage because we are sending a
1464                 // callback even in the successful stop case. If stopRecognition is successful,
1465                 // suggested next action RESTART_RECOGNITION will be sent.
1466                 // TODO(b/281608561): This code path will be removed with other enrollment flows in
1467                 //  this class.
1468                 try {
1469                     int result = stopRecognitionLocked();
1470                     if (result == STATUS_OK) {
1471                         sendSoundTriggerFailure(new SoundTriggerFailure(ERROR_CODE_UNKNOWN,
1472                                 "stopped recognition because of enrollment update",
1473                                 FailureSuggestedAction.RESTART_RECOGNITION));
1474                     }
1475                     // only log to logcat here because many failures can be false positives such as
1476                     // calling stopRecognition where there is no started session.
1477                     Log.w(TAG, "Failed to stop recognition after enrollment update: code="
1478                             + result);
1479                 } catch (Exception e) {
1480                     Slog.w(TAG, "Failed to stop recognition after enrollment update", e);
1481                     if (CompatChanges.isChangeEnabled(SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS)) {
1482                         sendSoundTriggerFailure(new SoundTriggerFailure(ERROR_CODE_UNKNOWN,
1483                                 "Failed to stop recognition after enrollment update: "
1484                                         + Log.getStackTraceString(e),
1485                                 FailureSuggestedAction.RECREATE_DETECTOR));
1486                     } else {
1487                         updateAndNotifyStateChangedLocked(STATE_ERROR);
1488                     }
1489                     return;
1490                 }
1491             }
1492 
1493             // Execute a refresh availability task - which should then notify of a change.
1494             new RefreshAvailabilityTask().execute();
1495         }
1496     }
1497 
1498     @GuardedBy("mLock")
1499     @SuppressWarnings("FlaggedApi") // RecognitionConfig.Builder is available internally.
startRecognitionLocked(int recognitionFlags, @NonNull byte[] data)1500     private int startRecognitionLocked(int recognitionFlags, @NonNull byte[] data) {
1501         if (DBG) {
1502             Slog.d(TAG, "startRecognition("
1503                     + recognitionFlags
1504                     + ", " + Arrays.toString(data) + ")");
1505         }
1506         if (mAvailability == STATE_INVALID || mAvailability == STATE_ERROR) {
1507             throw new IllegalStateException(
1508                     "startRecognition called on an invalid detector or error state");
1509         }
1510 
1511         // Check if we can start/stop a recognition.
1512         if (mAvailability != STATE_KEYPHRASE_ENROLLED) {
1513             throw new UnsupportedOperationException(
1514                     "Recognition for the given keyphrase is not supported");
1515         }
1516 
1517         List<KeyphraseRecognitionExtra> recognitionExtra =
1518             new ArrayList<KeyphraseRecognitionExtra>(1);
1519         // TODO: Do we need to do something about the confidence level here?
1520         recognitionExtra.add(new KeyphraseRecognitionExtra(mKeyphraseMetadata.getId(),
1521             mKeyphraseMetadata.getRecognitionModeFlags(), 0, new ConfidenceLevel[0]));
1522         boolean captureTriggerAudio =
1523                 (recognitionFlags&RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0;
1524         boolean allowMultipleTriggers =
1525                 (recognitionFlags&RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0;
1526         boolean runInBatterySaver = (recognitionFlags&RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER) != 0;
1527 
1528         int audioCapabilities = 0;
1529         if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION) != 0) {
1530             audioCapabilities |= AUDIO_CAPABILITY_ECHO_CANCELLATION;
1531         }
1532         if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION) != 0) {
1533             audioCapabilities |= AUDIO_CAPABILITY_NOISE_SUPPRESSION;
1534         }
1535 
1536         int code;
1537         try {
1538             code = mSoundTriggerSession.startRecognition(
1539                 mKeyphraseMetadata.getId(),
1540                 mLocale.toLanguageTag(),
1541                 mInternalCallback,
1542                 new RecognitionConfig.Builder()
1543                     .setCaptureRequested(captureTriggerAudio)
1544                     .setMultipleTriggersAllowed(allowMultipleTriggers)
1545                     .setKeyphrases(recognitionExtra)
1546                     .setData(data)
1547                     .setAudioCapabilities(audioCapabilities)
1548                     .build(),
1549                 runInBatterySaver);
1550         } catch (RemoteException e) {
1551             throw e.rethrowFromSystemServer();
1552         }
1553 
1554         if (code != STATUS_OK) {
1555             Slog.w(TAG, "startRecognition() failed with error code " + code);
1556         }
1557         return code;
1558     }
1559 
1560     @GuardedBy("mLock")
stopRecognitionLocked()1561     private int stopRecognitionLocked() {
1562         int code;
1563         try {
1564             code = mSoundTriggerSession.stopRecognition(mKeyphraseMetadata.getId(),
1565                     mInternalCallback);
1566         } catch (RemoteException e) {
1567             throw e.rethrowFromSystemServer();
1568         }
1569 
1570         if (code != STATUS_OK) {
1571             Slog.w(TAG, "stopRecognition() failed with error code " + code);
1572         }
1573         return code;
1574     }
1575 
1576     @GuardedBy("mLock")
setParameterLocked(@odelParams int modelParam, int value)1577     private int setParameterLocked(@ModelParams int modelParam, int value) {
1578         try {
1579             int code = mSoundTriggerSession.setParameter(mKeyphraseMetadata.getId(), modelParam,
1580                     value);
1581 
1582             if (code != STATUS_OK) {
1583                 Slog.w(TAG, "setParameter failed with error code " + code);
1584             }
1585 
1586             return code;
1587         } catch (RemoteException e) {
1588             throw e.rethrowFromSystemServer();
1589         }
1590     }
1591 
1592     @GuardedBy("mLock")
getParameterLocked(@odelParams int modelParam)1593     private int getParameterLocked(@ModelParams int modelParam) {
1594         try {
1595             return mSoundTriggerSession.getParameter(mKeyphraseMetadata.getId(), modelParam);
1596         } catch (RemoteException e) {
1597             throw e.rethrowFromSystemServer();
1598         }
1599     }
1600 
1601     @GuardedBy("mLock")
1602     @Nullable
queryParameterLocked(@odelParams int modelParam)1603     private ModelParamRange queryParameterLocked(@ModelParams int modelParam) {
1604         try {
1605             SoundTrigger.ModelParamRange modelParamRange =
1606                     mSoundTriggerSession.queryParameter(mKeyphraseMetadata.getId(), modelParam);
1607 
1608             if (modelParamRange == null) {
1609                 return null;
1610             }
1611 
1612             return new ModelParamRange(modelParamRange);
1613         } catch (RemoteException e) {
1614             throw e.rethrowFromSystemServer();
1615         }
1616     }
1617 
1618     @GuardedBy("mLock")
updateAndNotifyStateChangedLocked(int availability)1619     private void updateAndNotifyStateChangedLocked(int availability) {
1620         updateAvailabilityLocked(availability);
1621         notifyStateChangedLocked();
1622     }
1623 
1624     @GuardedBy("mLock")
updateAvailabilityLocked(int availability)1625     private void updateAvailabilityLocked(int availability) {
1626         if (DBG) {
1627             Slog.d(TAG, "Hotword availability changed from " + mAvailability
1628                     + " -> " + availability);
1629         }
1630         if (!mIsAvailabilityOverriddenByTestApi) {
1631             mAvailability = availability;
1632         }
1633     }
1634 
1635     @GuardedBy("mLock")
notifyStateChangedLocked()1636     private void notifyStateChangedLocked() {
1637         Message message = Message.obtain(mHandler, MSG_AVAILABILITY_CHANGED);
1638         message.arg1 = mAvailability;
1639         message.sendToTarget();
1640     }
1641 
1642     @GuardedBy("mLock")
sendUnknownFailure(String failureMessage)1643     private void sendUnknownFailure(String failureMessage) {
1644         // update but do not call onAvailabilityChanged callback for STATE_ERROR
1645         updateAvailabilityLocked(STATE_ERROR);
1646         Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE, failureMessage).sendToTarget();
1647     }
1648 
sendSoundTriggerFailure(@onNull SoundTriggerFailure soundTriggerFailure)1649     private void sendSoundTriggerFailure(@NonNull SoundTriggerFailure soundTriggerFailure) {
1650         Message.obtain(mHandler, MSG_DETECTION_SOUND_TRIGGER_FAILURE, soundTriggerFailure)
1651                 .sendToTarget();
1652     }
1653 
1654     /** @hide */
1655     static final class SoundTriggerListener extends IHotwordRecognitionStatusCallback.Stub {
1656         private final Handler mHandler;
1657 
SoundTriggerListener(Handler handler)1658         public SoundTriggerListener(Handler handler) {
1659             mHandler = handler;
1660         }
1661 
1662         @Override
onKeyphraseDetected( KeyphraseRecognitionEvent event, HotwordDetectedResult result)1663         public void onKeyphraseDetected(
1664                 KeyphraseRecognitionEvent event, HotwordDetectedResult result) {
1665             if (DBG) {
1666                 Slog.d(TAG, "onDetected(" + event + ")");
1667             } else {
1668                 Slog.i(TAG, "onDetected");
1669             }
1670             Message.obtain(mHandler, MSG_HOTWORD_DETECTED,
1671                     new EventPayload.Builder(event)
1672                             .setHotwordDetectedResult(result)
1673                             .build())
1674                     .sendToTarget();
1675         }
1676 
1677         @Override
onKeyphraseDetectedFromExternalSource(HotwordDetectedResult result)1678         public void onKeyphraseDetectedFromExternalSource(HotwordDetectedResult result) {
1679             Slog.i(TAG, "onKeyphraseDetectedFromExternalSource");
1680             EventPayload.Builder eventPayloadBuilder = new EventPayload.Builder();
1681             if (android.app.wearable.Flags.enableHotwordWearableSensingApi()) {
1682                 eventPayloadBuilder.setIsRecognitionStopped(false);
1683             }
1684             Message.obtain(
1685                             mHandler,
1686                             MSG_HOTWORD_DETECTED,
1687                             eventPayloadBuilder.setHotwordDetectedResult(result).build())
1688                     .sendToTarget();
1689         }
1690 
1691         @Override
onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event)1692         public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) {
1693             Slog.w(TAG, "Generic sound trigger event detected at AOHD: " + event);
1694         }
1695 
1696         @Override
onRejected(@onNull HotwordRejectedResult result)1697         public void onRejected(@NonNull HotwordRejectedResult result) {
1698             if (DBG) {
1699                 Slog.d(TAG, "onRejected(" + result + ")");
1700             } else {
1701                 Slog.i(TAG, "onRejected");
1702             }
1703             Message.obtain(mHandler, MSG_HOTWORD_REJECTED, result).sendToTarget();
1704         }
1705 
1706         @Override
onHotwordDetectionServiceFailure( HotwordDetectionServiceFailure hotwordDetectionServiceFailure)1707         public void onHotwordDetectionServiceFailure(
1708                 HotwordDetectionServiceFailure hotwordDetectionServiceFailure) {
1709             Slog.v(TAG, "onHotwordDetectionServiceFailure: " + hotwordDetectionServiceFailure);
1710             if (hotwordDetectionServiceFailure != null) {
1711                 Message.obtain(mHandler, MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE,
1712                         hotwordDetectionServiceFailure).sendToTarget();
1713             } else {
1714                 Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE,
1715                         "Error data is null").sendToTarget();
1716             }
1717         }
1718 
1719         @Override
onVisualQueryDetectionServiceFailure( VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure)1720         public void onVisualQueryDetectionServiceFailure(
1721                 VisualQueryDetectionServiceFailure visualQueryDetectionServiceFailure)
1722                 throws RemoteException {
1723             // It should never be called here.
1724             Slog.w(TAG,
1725                     "onVisualQueryDetectionServiceFailure: " + visualQueryDetectionServiceFailure);
1726         }
1727 
1728         @Override
onSoundTriggerFailure(SoundTriggerFailure soundTriggerFailure)1729         public void onSoundTriggerFailure(SoundTriggerFailure soundTriggerFailure) {
1730             Message.obtain(mHandler, MSG_DETECTION_SOUND_TRIGGER_FAILURE,
1731                     Objects.requireNonNull(soundTriggerFailure)).sendToTarget();
1732         }
1733 
1734         @Override
onUnknownFailure(String errorMessage)1735         public void onUnknownFailure(String errorMessage) throws RemoteException {
1736             Slog.v(TAG, "onUnknownFailure: " + errorMessage);
1737             Message.obtain(mHandler, MSG_DETECTION_UNKNOWN_FAILURE,
1738                     !TextUtils.isEmpty(errorMessage) ? errorMessage
1739                             : "Error data is null").sendToTarget();
1740         }
1741 
1742         @Override
onRecognitionPaused()1743         public void onRecognitionPaused() {
1744             Slog.i(TAG, "onRecognitionPaused");
1745             mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE);
1746         }
1747 
1748         @Override
onRecognitionResumed()1749         public void onRecognitionResumed() {
1750             Slog.i(TAG, "onRecognitionResumed");
1751             mHandler.sendEmptyMessage(MSG_DETECTION_RESUME);
1752         }
1753 
1754         @Override
onStatusReported(int status)1755         public void onStatusReported(int status) {
1756             if (DBG) {
1757                 Slog.d(TAG, "onStatusReported(" + status + ")");
1758             } else {
1759                 Slog.i(TAG, "onStatusReported");
1760             }
1761             Message message = Message.obtain(mHandler, MSG_HOTWORD_STATUS_REPORTED);
1762             message.arg1 = status;
1763             message.sendToTarget();
1764         }
1765 
1766         @Override
onProcessRestarted()1767         public void onProcessRestarted() {
1768             Slog.i(TAG, "onProcessRestarted");
1769             mHandler.sendEmptyMessage(MSG_PROCESS_RESTARTED);
1770         }
1771 
1772         @Override
onOpenFile(String filename, AndroidFuture future)1773         public void onOpenFile(String filename, AndroidFuture future) throws RemoteException {
1774             throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
1775         }
1776     }
1777 
onDetectorRemoteException()1778     void onDetectorRemoteException() {
1779         Message.obtain(mHandler, MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE,
1780                 new HotwordDetectionServiceFailure(
1781                         HotwordDetectionServiceFailure.ERROR_CODE_REMOTE_EXCEPTION,
1782                         "Detector remote exception occurs")).sendToTarget();
1783     }
1784 
1785     class MyHandler extends Handler {
MyHandler(@onNull Looper looper)1786         MyHandler(@NonNull Looper looper) {
1787             super(looper);
1788         }
1789 
1790         @Override
handleMessage(Message msg)1791         public void handleMessage(Message msg) {
1792             synchronized (mLock) {
1793                 if (mAvailability == STATE_INVALID) {
1794                     Slog.w(TAG, "Received message: " + msg.what + " for an invalid detector");
1795                     return;
1796                 }
1797             }
1798             final Message message = Message.obtain(msg);
1799             Binder.withCleanCallingIdentity(() -> mExternalExecutor.execute(() -> {
1800                 Slog.i(TAG, "handle message " + message.what);
1801                 switch (message.what) {
1802                     case MSG_AVAILABILITY_CHANGED:
1803                         mExternalCallback.onAvailabilityChanged(message.arg1);
1804                         break;
1805                     case MSG_HOTWORD_DETECTED:
1806                         mExternalCallback.onDetected((EventPayload) message.obj);
1807                         break;
1808                     case MSG_DETECTION_ERROR:
1809                         // TODO(b/271534248): After reverting the workaround, this logic is still
1810                         // necessary.
1811                         mExternalCallback.onError();
1812                         break;
1813                     case MSG_DETECTION_PAUSE:
1814                         mExternalCallback.onRecognitionPaused();
1815                         break;
1816                     case MSG_DETECTION_RESUME:
1817                         mExternalCallback.onRecognitionResumed();
1818                         break;
1819                     case MSG_HOTWORD_REJECTED:
1820                         mExternalCallback.onRejected((HotwordRejectedResult) message.obj);
1821                         break;
1822                     case MSG_HOTWORD_STATUS_REPORTED:
1823                         mExternalCallback.onHotwordDetectionServiceInitialized(message.arg1);
1824                         break;
1825                     case MSG_PROCESS_RESTARTED:
1826                         mExternalCallback.onHotwordDetectionServiceRestarted();
1827                         break;
1828                     case MSG_DETECTION_HOTWORD_DETECTION_SERVICE_FAILURE:
1829                         mExternalCallback.onFailure((HotwordDetectionServiceFailure) message.obj);
1830                         break;
1831                     case MSG_DETECTION_SOUND_TRIGGER_FAILURE:
1832                         mExternalCallback.onFailure((SoundTriggerFailure) message.obj);
1833                         break;
1834                     case MSG_DETECTION_UNKNOWN_FAILURE:
1835                         mExternalCallback.onUnknownFailure((String) message.obj);
1836                         break;
1837                     default:
1838                         super.handleMessage(message);
1839                 }
1840                 message.recycle();
1841             }));
1842         }
1843     }
1844 
1845     // TODO(b/267681692): remove the AsyncTask usage
1846     class RefreshAvailabilityTask extends AsyncTask<Void, Void, Void> {
1847 
1848         @Override
doInBackground(Void... params)1849         public Void doInBackground(Void... params) {
1850             try {
1851                 int availability = internalGetInitialAvailability();
1852 
1853                 synchronized (mLock) {
1854                     if (availability == STATE_NOT_READY) {
1855                         internalUpdateEnrolledKeyphraseMetadata();
1856                         if (mKeyphraseMetadata != null) {
1857                             availability = STATE_KEYPHRASE_ENROLLED;
1858                         } else {
1859                             availability = STATE_KEYPHRASE_UNENROLLED;
1860                         }
1861                     }
1862                     updateAndNotifyStateChangedLocked(availability);
1863                 }
1864             } catch (Exception e) {
1865                 // Any exception here not caught will crash the process because AsyncTask does not
1866                 // bubble up the exceptions to the client app, so we must propagate it to the app.
1867                 Slog.w(TAG, "Failed to refresh availability", e);
1868                 synchronized (mLock) {
1869                     if (CompatChanges.isChangeEnabled(SEND_ON_FAILURE_FOR_ASYNC_EXCEPTIONS)) {
1870                         sendUnknownFailure(
1871                                 "Failed to refresh availability: " + Log.getStackTraceString(e));
1872                     } else {
1873                         updateAndNotifyStateChangedLocked(STATE_ERROR);
1874                     }
1875                 }
1876             }
1877 
1878             return null;
1879         }
1880 
1881         /**
1882          * @return The initial availability without checking the enrollment status.
1883          */
internalGetInitialAvailability()1884         private int internalGetInitialAvailability() {
1885             synchronized (mLock) {
1886                 // This detector has already been invalidated.
1887                 if (mAvailability == STATE_INVALID) {
1888                     return STATE_INVALID;
1889                 }
1890             }
1891 
1892             if (!CompatChanges.isChangeEnabled(THROW_ON_INITIALIZE_IF_NO_DSP)) {
1893                 ModuleProperties dspModuleProperties;
1894                 try {
1895                     dspModuleProperties =
1896                             mSoundTriggerSession.getDspModuleProperties();
1897                 } catch (RemoteException e) {
1898                     throw e.rethrowFromSystemServer();
1899                 }
1900 
1901                 // No DSP available
1902                 if (dspModuleProperties == null) {
1903                     return STATE_HARDWARE_UNAVAILABLE;
1904                 }
1905             }
1906 
1907             return STATE_NOT_READY;
1908         }
1909 
internalUpdateEnrolledKeyphraseMetadata()1910         private void internalUpdateEnrolledKeyphraseMetadata() {
1911             try {
1912                 mKeyphraseMetadata = mModelManagementService.getEnrolledKeyphraseMetadata(
1913                         mText, mLocale.toLanguageTag());
1914             } catch (RemoteException e) {
1915                 throw e.rethrowFromSystemServer();
1916             }
1917         }
1918     }
1919 
1920     @Override
equals(Object obj)1921     public boolean equals(Object obj) {
1922         if (CompatChanges.isChangeEnabled(MULTIPLE_ACTIVE_HOTWORD_DETECTORS)) {
1923             if (!(obj instanceof AlwaysOnHotwordDetector)) {
1924                 return false;
1925             }
1926             AlwaysOnHotwordDetector other = (AlwaysOnHotwordDetector) obj;
1927             return TextUtils.equals(mText, other.mText) && mLocale.equals(other.mLocale);
1928         }
1929 
1930         return super.equals(obj);
1931     }
1932 
1933     @Override
hashCode()1934     public int hashCode() {
1935         return Objects.hash(mText, mLocale);
1936     }
1937 
1938     /** @hide */
1939     @Override
dump(String prefix, PrintWriter pw)1940     public void dump(String prefix, PrintWriter pw) {
1941         synchronized (mLock) {
1942             pw.print(prefix); pw.print("Text="); pw.println(mText);
1943             pw.print(prefix); pw.print("Locale="); pw.println(mLocale);
1944             pw.print(prefix); pw.print("Availability="); pw.println(mAvailability);
1945             pw.print(prefix); pw.print("KeyphraseMetadata="); pw.println(mKeyphraseMetadata);
1946             pw.print(prefix); pw.print("EnrollmentInfo="); pw.println(mKeyphraseEnrollmentInfo);
1947         }
1948     }
1949 }
1950