• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
5  * use this file except in compliance with the License. You may obtain a copy of
6  * the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13  * License for the specific language governing permissions and limitations under
14  * the License.
15  */
16 
17 package android.speech;
18 
19 import android.Manifest;
20 import android.annotation.NonNull;
21 import android.annotation.Nullable;
22 import android.annotation.SdkConstant;
23 import android.annotation.SdkConstant.SdkConstantType;
24 import android.annotation.SuppressLint;
25 import android.annotation.TestApi;
26 import android.app.AppOpsManager;
27 import android.app.Service;
28 import android.content.AttributionSource;
29 import android.content.Context;
30 import android.content.ContextParams;
31 import android.content.Intent;
32 import android.content.PermissionChecker;
33 import android.os.Binder;
34 import android.os.Bundle;
35 import android.os.Handler;
36 import android.os.IBinder;
37 import android.os.Looper;
38 import android.os.Message;
39 import android.os.RemoteException;
40 import android.util.Log;
41 
42 import com.android.internal.annotations.GuardedBy;
43 import com.android.internal.util.function.pooled.PooledLambda;
44 
45 import java.lang.ref.WeakReference;
46 import java.util.HashMap;
47 import java.util.Map;
48 import java.util.Objects;
49 
50 /**
51  * This class provides a base class for recognition service implementations. This class should be
52  * extended only in case you wish to implement a new speech recognizer. Please note that the
53  * implementation of this service is stateless.
54  */
55 public abstract class RecognitionService extends Service {
56     /**
57      * The {@link Intent} that must be declared as handled by the service.
58      */
59     @SdkConstant(SdkConstantType.SERVICE_ACTION)
60     public static final String SERVICE_INTERFACE = "android.speech.RecognitionService";
61 
62     /**
63      * Name under which a RecognitionService component publishes information about itself.
64      * This meta-data should reference an XML resource containing a
65      * <code>&lt;{@link android.R.styleable#RecognitionService recognition-service}&gt;</code> or
66      * <code>&lt;{@link android.R.styleable#RecognitionService on-device-recognition-service}
67      * &gt;</code> tag.
68      */
69     public static final String SERVICE_META_DATA = "android.speech";
70 
71     /** Log messages identifier */
72     private static final String TAG = "RecognitionService";
73 
74     /** Debugging flag */
75     private static final boolean DBG = false;
76 
77     private static final int DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT = 1;
78 
79     private final Map<IBinder, SessionState> mSessions = new HashMap<>();
80 
81     /** Binder of the recognition service */
82     private final RecognitionServiceBinder mBinder = new RecognitionServiceBinder(this);
83 
84     private static final int MSG_START_LISTENING = 1;
85 
86     private static final int MSG_STOP_LISTENING = 2;
87 
88     private static final int MSG_CANCEL = 3;
89 
90     private static final int MSG_RESET = 4;
91 
92     private static final int MSG_CHECK_RECOGNITION_SUPPORT = 5;
93 
94     private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 6;
95 
96     private final Handler mHandler = new Handler() {
97         @Override
98         public void handleMessage(Message msg) {
99             switch (msg.what) {
100                 case MSG_START_LISTENING:
101                     StartListeningArgs args = (StartListeningArgs) msg.obj;
102                     dispatchStartListening(args.mIntent, args.mListener, args.mAttributionSource);
103                     break;
104                 case MSG_STOP_LISTENING:
105                     dispatchStopListening((IRecognitionListener) msg.obj);
106                     break;
107                 case MSG_CANCEL:
108                     dispatchCancel((IRecognitionListener) msg.obj);
109                     break;
110                 case MSG_RESET:
111                     dispatchClearCallback((IRecognitionListener) msg.obj);
112                     break;
113                 case MSG_CHECK_RECOGNITION_SUPPORT:
114                     CheckRecognitionSupportArgs checkArgs = (CheckRecognitionSupportArgs) msg.obj;
115                     dispatchCheckRecognitionSupport(
116                             checkArgs.mIntent, checkArgs.callback, checkArgs.mAttributionSource);
117                     break;
118                 case MSG_TRIGGER_MODEL_DOWNLOAD:
119                     ModelDownloadArgs modelDownloadArgs = (ModelDownloadArgs) msg.obj;
120                     dispatchTriggerModelDownload(
121                             modelDownloadArgs.mIntent,
122                             modelDownloadArgs.mAttributionSource,
123                             modelDownloadArgs.mListener);
124                     break;
125             }
126         }
127     };
128 
dispatchStartListening(Intent intent, final IRecognitionListener listener, @NonNull AttributionSource attributionSource)129     private void dispatchStartListening(Intent intent, final IRecognitionListener listener,
130             @NonNull AttributionSource attributionSource) {
131         Callback currentCallback = null;
132         SessionState sessionState = mSessions.get(listener.asBinder());
133 
134         try {
135             if (sessionState == null) {
136                 if (mSessions.size() >= getMaxConcurrentSessionsCount()) {
137                     listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY);
138                     Log.i(TAG, "#startListening received "
139                             + "when the service's capacity is full - ignoring this call.");
140                     return;
141                 }
142 
143                 boolean preflightPermissionCheckPassed =
144                         intent.hasExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE)
145                                 || checkPermissionForPreflightNotHardDenied(attributionSource);
146                 if (preflightPermissionCheckPassed) {
147                     currentCallback = new Callback(listener, attributionSource);
148                     sessionState = new SessionState(currentCallback);
149                     mSessions.put(listener.asBinder(), sessionState);
150                     if (DBG) {
151                         Log.d(TAG, "Added a new session to the map, pending permission checks");
152                     }
153                     RecognitionService.this.onStartListening(intent, currentCallback);
154                 }
155 
156                 if (!preflightPermissionCheckPassed
157                         || !checkPermissionAndStartDataDelivery(sessionState)) {
158                     listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS);
159                     if (preflightPermissionCheckPassed) {
160                         // If start listening was attempted, cancel the callback.
161                         RecognitionService.this.onCancel(currentCallback);
162                         mSessions.remove(listener.asBinder());
163                         finishDataDelivery(sessionState);
164                         sessionState.reset();
165                     }
166                     Log.i(TAG, "#startListening received from a caller "
167                             + "without permission " + Manifest.permission.RECORD_AUDIO + ".");
168                 }
169             } else {
170                 listener.onError(SpeechRecognizer.ERROR_CLIENT);
171                 Log.i(TAG, "#startListening received "
172                         + "for a listener which is already in session - ignoring this call.");
173             }
174         } catch (RemoteException e) {
175             Log.d(TAG, "#onError call from #startListening failed.");
176         }
177     }
178 
dispatchStopListening(IRecognitionListener listener)179     private void dispatchStopListening(IRecognitionListener listener) {
180         SessionState sessionState = mSessions.get(listener.asBinder());
181         if (sessionState == null) {
182             try {
183                 listener.onError(SpeechRecognizer.ERROR_CLIENT);
184             } catch (RemoteException e) {
185                 Log.d(TAG, "#onError call from #stopListening failed.");
186             }
187             Log.w(TAG, "#stopListening received for a listener "
188                     + "which has not started a session - ignoring this call.");
189         } else {
190             RecognitionService.this.onStopListening(sessionState.mCallback);
191         }
192     }
193 
dispatchCancel(IRecognitionListener listener)194     private void dispatchCancel(IRecognitionListener listener) {
195         SessionState sessionState = mSessions.get(listener.asBinder());
196         if (sessionState == null) {
197             Log.w(TAG, "#cancel received for a listener which has not started a session "
198                     + "- ignoring this call.");
199         } else {
200             RecognitionService.this.onCancel(sessionState.mCallback);
201             dispatchClearCallback(listener);
202         }
203     }
204 
dispatchClearCallback(IRecognitionListener listener)205     private void dispatchClearCallback(IRecognitionListener listener) {
206         SessionState sessionState = mSessions.remove(listener.asBinder());
207         if (sessionState != null) {
208             if (DBG) {
209                 Log.d(TAG, "Removed session from the map for listener = "
210                         + listener.asBinder() + ".");
211             }
212             finishDataDelivery(sessionState);
213             sessionState.reset();
214         }
215     }
216 
dispatchCheckRecognitionSupport( Intent intent, IRecognitionSupportCallback callback, AttributionSource attributionSource)217     private void dispatchCheckRecognitionSupport(
218             Intent intent, IRecognitionSupportCallback callback,
219             AttributionSource attributionSource) {
220         RecognitionService.this.onCheckRecognitionSupport(
221                 intent,
222                 attributionSource,
223                 new SupportCallback(callback));
224     }
225 
dispatchTriggerModelDownload( Intent intent, AttributionSource attributionSource, IModelDownloadListener listener)226     private void dispatchTriggerModelDownload(
227             Intent intent,
228             AttributionSource attributionSource,
229             IModelDownloadListener listener) {
230         if (listener == null) {
231             RecognitionService.this.onTriggerModelDownload(intent, attributionSource);
232         } else {
233             RecognitionService.this.onTriggerModelDownload(
234                     intent,
235                     attributionSource,
236                     new ModelDownloadListener() {
237 
238                         private final Object mLock = new Object();
239 
240                         @GuardedBy("mLock")
241                         private boolean mIsTerminated = false;
242 
243                         @Override
244                         public void onProgress(int completedPercent) {
245                             synchronized (mLock) {
246                                 if (mIsTerminated) {
247                                     return;
248                                 }
249                                 try {
250                                     listener.onProgress(completedPercent);
251                                 } catch (RemoteException e) {
252                                     throw e.rethrowFromSystemServer();
253                                 }
254                             }
255                         }
256 
257                         @Override
258                         public void onSuccess() {
259                             synchronized (mLock) {
260                                 if (mIsTerminated) {
261                                     return;
262                                 }
263                                 mIsTerminated = true;
264                                 try {
265                                     listener.onSuccess();
266                                 } catch (RemoteException e) {
267                                     throw e.rethrowFromSystemServer();
268                                 }
269                             }
270                         }
271 
272                         @Override
273                         public void onScheduled() {
274                             synchronized (mLock) {
275                                 if (mIsTerminated) {
276                                     return;
277                                 }
278                                 mIsTerminated = true;
279                                 try {
280                                     listener.onScheduled();
281                                 } catch (RemoteException e) {
282                                     throw e.rethrowFromSystemServer();
283                                 }
284                             }
285                         }
286 
287                         @Override
288                         public void onError(int error) {
289                             synchronized (mLock) {
290                                 if (mIsTerminated) {
291                                     return;
292                                 }
293                                 mIsTerminated = true;
294                                 try {
295                                     listener.onError(error);
296                                 } catch (RemoteException e) {
297                                     throw e.rethrowFromSystemServer();
298                                 }
299                             }
300                         }
301                     });
302         }
303     }
304 
305     private static class StartListeningArgs {
306         public final Intent mIntent;
307 
308         public final IRecognitionListener mListener;
309         @NonNull public final AttributionSource mAttributionSource;
310 
StartListeningArgs(Intent intent, IRecognitionListener listener, @NonNull AttributionSource attributionSource)311         public StartListeningArgs(Intent intent, IRecognitionListener listener,
312                 @NonNull AttributionSource attributionSource) {
313             this.mIntent = intent;
314             this.mListener = listener;
315             this.mAttributionSource = attributionSource;
316         }
317     }
318 
319     private static class CheckRecognitionSupportArgs {
320         public final Intent mIntent;
321         public final IRecognitionSupportCallback callback;
322         public final AttributionSource mAttributionSource;
323 
CheckRecognitionSupportArgs( Intent intent, IRecognitionSupportCallback callback, AttributionSource attributionSource)324         private CheckRecognitionSupportArgs(
325                 Intent intent,
326                 IRecognitionSupportCallback callback,
327                 AttributionSource attributionSource) {
328             this.mIntent = intent;
329             this.callback = callback;
330             this.mAttributionSource = attributionSource;
331         }
332     }
333 
334     private static class ModelDownloadArgs {
335         final Intent mIntent;
336         final AttributionSource mAttributionSource;
337         @Nullable final IModelDownloadListener mListener;
338 
ModelDownloadArgs( Intent intent, AttributionSource attributionSource, @Nullable IModelDownloadListener listener)339         private ModelDownloadArgs(
340                 Intent intent,
341                 AttributionSource attributionSource,
342                 @Nullable IModelDownloadListener listener) {
343             this.mIntent = intent;
344             this.mAttributionSource = attributionSource;
345             this.mListener = listener;
346         }
347     }
348 
349     /**
350      * Notifies the service that it should start listening for speech.
351      *
352      * <p> If you are recognizing speech from the microphone, in this callback you
353      * should create an attribution context for the caller such that when you access
354      * the mic the caller would be properly blamed (and their permission checked in
355      * the process) for accessing the microphone and that you served as a proxy for
356      * this sensitive data (and your permissions would be checked in the process).
357      * You should also open the mic in this callback via the attribution context
358      * and close the mic before returning the recognized result. If you don't do
359      * that then the caller would be blamed and you as being a proxy as well as you
360      * would get one more blame on yourself when you open the microphone.
361      *
362      * <pre>
363      * Context attributionContext = context.createContext(new ContextParams.Builder()
364      *     .setNextAttributionSource(callback.getCallingAttributionSource())
365      *     .build());
366      *
367      * AudioRecord recorder = AudioRecord.Builder()
368      *     .setContext(attributionContext);
369      *     . . .
370      *    .build();
371      *
372      * recorder.startRecording()
373      * </pre>
374      *
375      * @param recognizerIntent contains parameters for the recognition to be performed. The intent
376      *        may also contain optional extras, see {@link RecognizerIntent}. If these values are
377      *        not set explicitly, default values should be used by the recognizer.
378      * @param listener that will receive the service's callbacks
379      */
onStartListening(Intent recognizerIntent, Callback listener)380     protected abstract void onStartListening(Intent recognizerIntent, Callback listener);
381 
382     /**
383      * Notifies the service that it should cancel the speech recognition.
384      */
onCancel(Callback listener)385     protected abstract void onCancel(Callback listener);
386 
387     /**
388      * Notifies the service that it should stop listening for speech. Speech captured so far should
389      * be recognized as if the user had stopped speaking at this point. This method is only called
390      * if the application calls it explicitly.
391      */
onStopListening(Callback listener)392     protected abstract void onStopListening(Callback listener);
393 
394     /**
395      * Queries the service on whether it would support a {@link #onStartListening(Intent, Callback)}
396      * for the same {@code recognizerIntent}.
397      *
398      * <p>The service will notify the caller about the level of support or error via
399      * {@link SupportCallback}.
400      *
401      * <p>If the service does not offer the support check it will notify the caller with
402      * {@link SpeechRecognizer#ERROR_CANNOT_CHECK_SUPPORT}.
403      */
onCheckRecognitionSupport( @onNull Intent recognizerIntent, @NonNull SupportCallback supportCallback)404     public void onCheckRecognitionSupport(
405             @NonNull Intent recognizerIntent,
406             @NonNull SupportCallback supportCallback) {
407         if (DBG) {
408             Log.i(TAG, String.format("#onSupports [%s]", recognizerIntent));
409         }
410         supportCallback.onError(SpeechRecognizer.ERROR_CANNOT_CHECK_SUPPORT);
411     }
412 
413     /**
414      * Queries the service on whether it would support a {@link #onStartListening(Intent, Callback)}
415      * for the same {@code recognizerIntent}.
416      *
417      * <p>The service will notify the caller about the level of support or error via
418      * {@link SupportCallback}.
419      *
420      * <p>If the service does not offer the support check it will notify the caller with
421      * {@link SpeechRecognizer#ERROR_CANNOT_CHECK_SUPPORT}.
422      *
423      * <p>Provides the calling AttributionSource to the service implementation so that permissions
424      * and bandwidth could be correctly blamed.</p>
425      */
onCheckRecognitionSupport( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource, @NonNull SupportCallback supportCallback)426     public void onCheckRecognitionSupport(
427             @NonNull Intent recognizerIntent,
428             @NonNull AttributionSource attributionSource,
429             @NonNull SupportCallback supportCallback) {
430         onCheckRecognitionSupport(recognizerIntent, supportCallback);
431     }
432 
433     /**
434      * Requests the download of the recognizer support for {@code recognizerIntent}.
435      */
onTriggerModelDownload(@onNull Intent recognizerIntent)436     public void onTriggerModelDownload(@NonNull Intent recognizerIntent) {
437         if (DBG) {
438             Log.i(TAG, String.format("#downloadModel [%s]", recognizerIntent));
439         }
440     }
441 
442     /**
443      * Requests the download of the recognizer support for {@code recognizerIntent}.
444      *
445      * <p>Provides the calling AttributionSource to the service implementation so that permissions
446      * and bandwidth could be correctly blamed.</p>
447      */
onTriggerModelDownload( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource)448     public void onTriggerModelDownload(
449             @NonNull Intent recognizerIntent,
450             @NonNull AttributionSource attributionSource) {
451         onTriggerModelDownload(recognizerIntent);
452     }
453 
454     /**
455      * Requests the download of the recognizer support for {@code recognizerIntent}.
456      *
457      * <p> Provides the calling {@link AttributionSource} to the service implementation so that
458      * permissions and bandwidth could be correctly blamed.
459      *
460      * <p> Client will receive the progress updates via the given {@link ModelDownloadListener}:
461      *
462      * <li> If the model is already available, {@link ModelDownloadListener#onSuccess()} will be
463      * called directly. The model can be safely used afterwards.
464      *
465      * <li> If the {@link RecognitionService} has started the download,
466      * {@link ModelDownloadListener#onProgress(int)} will be called an unspecified (zero or more)
467      * number of times until the download is complete.
468      * When the download finishes, {@link ModelDownloadListener#onSuccess()} will be called.
469      * The model can be safely used afterwards.
470      *
471      * <li> If the {@link RecognitionService} has only scheduled the download, but won't satisfy it
472      * immediately, {@link ModelDownloadListener#onScheduled()} will be called.
473      * There will be no further updates on this listener.
474      *
475      * <li> If the request fails at any time due to a network or scheduling error,
476      * {@link ModelDownloadListener#onError(int)} will be called.
477      *
478      * @param recognizerIntent contains parameters for the recognition to be performed. The intent
479      *        may also contain optional extras, see {@link RecognizerIntent}.
480      * @param attributionSource the attribution source of the caller.
481      * @param listener on which to receive updates about the model download request.
482      */
onTriggerModelDownload( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource, @NonNull ModelDownloadListener listener)483     public void onTriggerModelDownload(
484             @NonNull Intent recognizerIntent,
485             @NonNull AttributionSource attributionSource,
486             @NonNull ModelDownloadListener listener) {
487         listener.onError(SpeechRecognizer.ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS);
488     }
489 
490     @Override
491     @SuppressLint("MissingNullability")
createContext(@onNull ContextParams contextParams)492     public Context createContext(@NonNull ContextParams contextParams) {
493         if (contextParams.getNextAttributionSource() != null) {
494             if (mHandler.getLooper().equals(Looper.myLooper())) {
495                 handleAttributionContextCreation(contextParams.getNextAttributionSource());
496             } else {
497                 mHandler.sendMessage(
498                         PooledLambda.obtainMessage(this::handleAttributionContextCreation,
499                                 contextParams.getNextAttributionSource()));
500             }
501         }
502         return super.createContext(contextParams);
503     }
504 
handleAttributionContextCreation(@onNull AttributionSource attributionSource)505     private void handleAttributionContextCreation(@NonNull AttributionSource attributionSource) {
506         for (SessionState sessionState : mSessions.values()) {
507             Callback currentCallback = sessionState.mCallback;
508             if (currentCallback != null
509                     && currentCallback.mCallingAttributionSource.equals(attributionSource)) {
510                 currentCallback.mAttributionContextCreated = true;
511             }
512         }
513     }
514 
515     @Override
onBind(final Intent intent)516     public final IBinder onBind(final Intent intent) {
517         if (DBG) Log.d(TAG, "#onBind, intent=" + intent);
518         onBindInternal();
519         return mBinder;
520     }
521 
522     /** @hide */
523     @SuppressLint("UnflaggedApi") // @TestApi without associated feature.
524     @TestApi
onBindInternal()525     public void onBindInternal() { }
526 
527     @Override
onDestroy()528     public void onDestroy() {
529         if (DBG) Log.d(TAG, "#onDestroy");
530         for (SessionState sessionState : mSessions.values()) {
531             finishDataDelivery(sessionState);
532             sessionState.reset();
533         }
534         mSessions.clear();
535         mBinder.clearReference();
536         super.onDestroy();
537     }
538 
539     /**
540      * Returns the maximal number of recognition sessions ongoing at the same time.
541      * <p>
542      * The default value is 1, meaning concurrency should be enabled by overriding this method.
543      */
getMaxConcurrentSessionsCount()544     public int getMaxConcurrentSessionsCount() {
545         return DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT;
546     }
547 
548     /**
549      * This class receives callbacks from the speech recognition service and forwards them to the
550      * user. An instance of this class is passed to the
551      * {@link RecognitionService#onStartListening(Intent, Callback)} method. Recognizers may call
552      * these methods on any thread.
553      */
554     public class Callback {
555         private final IRecognitionListener mListener;
556         @NonNull private final AttributionSource mCallingAttributionSource;
557         @Nullable private Context mAttributionContext;
558         private boolean mAttributionContextCreated;
559 
Callback(IRecognitionListener listener, @NonNull AttributionSource attributionSource)560         private Callback(IRecognitionListener listener,
561                 @NonNull AttributionSource attributionSource) {
562             mListener = listener;
563             mCallingAttributionSource = attributionSource;
564         }
565 
566         /**
567          * The service should call this method when the user has started to speak.
568          */
beginningOfSpeech()569         public void beginningOfSpeech() throws RemoteException {
570             mListener.onBeginningOfSpeech();
571         }
572 
573         /**
574          * The service should call this method when sound has been received. The purpose of this
575          * function is to allow giving feedback to the user regarding the captured audio.
576          *
577          * @param buffer a buffer containing a sequence of big-endian 16-bit integers representing a
578          *        single channel audio stream. The sample rate is implementation dependent.
579          */
bufferReceived(byte[] buffer)580         public void bufferReceived(byte[] buffer) throws RemoteException {
581             mListener.onBufferReceived(buffer);
582         }
583 
584         /**
585          * The service should call this method after the user stops speaking.
586          */
endOfSpeech()587         public void endOfSpeech() throws RemoteException {
588             mListener.onEndOfSpeech();
589         }
590 
591         /**
592          * The service should call this method when a network or recognition error occurred.
593          *
594          * @param error code is defined in {@link SpeechRecognizer}
595          */
error(@peechRecognizer.RecognitionError int error)596         public void error(@SpeechRecognizer.RecognitionError int error) throws RemoteException {
597             Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
598             mListener.onError(error);
599         }
600 
601         /**
602          * The service should call this method when partial recognition results are available. This
603          * method can be called at any time between {@link #beginningOfSpeech()} and
604          * {@link #results(Bundle)} when partial results are ready. This method may be called zero,
605          * one or multiple times for each call to {@link SpeechRecognizer#startListening(Intent)},
606          * depending on the speech recognition service implementation.
607          *
608          * @param partialResults the returned results. To retrieve the results in
609          *        ArrayList&lt;String&gt; format use {@link Bundle#getStringArrayList(String)} with
610          *        {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter
611          */
partialResults(Bundle partialResults)612         public void partialResults(Bundle partialResults) throws RemoteException {
613             mListener.onPartialResults(partialResults);
614         }
615 
616         /**
617          * The service should call this method when the endpointer is ready for the user to start
618          * speaking.
619          *
620          * @param params parameters set by the recognition service. Reserved for future use.
621          */
readyForSpeech(Bundle params)622         public void readyForSpeech(Bundle params) throws RemoteException {
623             mListener.onReadyForSpeech(params);
624         }
625 
626         /**
627          * The service should call this method when recognition results are ready.
628          *
629          * @param results the recognition results. To retrieve the results in {@code
630          *        ArrayList<String>} format use {@link Bundle#getStringArrayList(String)} with
631          *        {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter
632          */
results(Bundle results)633         public void results(Bundle results) throws RemoteException {
634             Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
635             mListener.onResults(results);
636         }
637 
638         /**
639          * The service should call this method when the sound level in the audio stream has changed.
640          * There is no guarantee that this method will be called.
641          *
642          * @param rmsdB the new RMS dB value
643          */
rmsChanged(float rmsdB)644         public void rmsChanged(float rmsdB) throws RemoteException {
645             mListener.onRmsChanged(rmsdB);
646         }
647 
648         /**
649          * The service should call this method for each ready segment of a long recognition session.
650          *
651          * @param results the recognition results. To retrieve the results in {@code
652          *        ArrayList<String>} format use {@link Bundle#getStringArrayList(String)} with
653          *        {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter
654          */
655         @SuppressLint({"CallbackMethodName", "RethrowRemoteException"})
segmentResults(@onNull Bundle results)656         public void segmentResults(@NonNull Bundle results) throws RemoteException {
657             mListener.onSegmentResults(results);
658         }
659 
660         /**
661          * The service should call this method to end a segmented session.
662          */
663         @SuppressLint({"CallbackMethodName", "RethrowRemoteException"})
endOfSegmentedSession()664         public void endOfSegmentedSession() throws RemoteException {
665             Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget();
666             mListener.onEndOfSegmentedSession();
667         }
668 
669         /**
670          * The service should call this method when the language detection (and switching)
671          * results are available. This method can be called on any number of occasions
672          * at any time between {@link #beginningOfSpeech()} and {@link #endOfSpeech()},
673          * depending on the speech recognition service implementation.
674          *
675          * @param results the returned language detection (and switching) results.
676          *        <p> To retrieve the most confidently detected language IETF tag
677          *        (as defined by BCP 47, e.g., "en-US", "de-DE"),
678          *        use {@link Bundle#getString(String)}
679          *        with {@link SpeechRecognizer#DETECTED_LANGUAGE} as the parameter.
680          *        <p> To retrieve the language detection confidence level represented by a value
681          *        prefixed by {@code LANGUAGE_DETECTION_CONFIDENCE_LEVEL_} defined in
682          *        {@link SpeechRecognizer}, use {@link Bundle#getInt(String)} with
683          *        {@link SpeechRecognizer#LANGUAGE_DETECTION_CONFIDENCE_LEVEL} as the parameter.
684          *        <p> To retrieve the alternative locales for the same language
685          *        retrieved by the key {@link SpeechRecognizer#DETECTED_LANGUAGE},
686          *        use {@link Bundle#getStringArrayList(String)}
687          *        with {@link SpeechRecognizer#TOP_LOCALE_ALTERNATIVES} as the parameter.
688          *        <p> To retrieve the language switching results represented by a value
689          *        prefixed by {@code LANGUAGE_SWITCH_RESULT_}
690          *        and defined in {@link SpeechRecognizer}, use {@link Bundle#getInt(String)}
691          *        with {@link SpeechRecognizer#LANGUAGE_SWITCH_RESULT} as the parameter.
692          */
693         @SuppressLint("CallbackMethodName") // For consistency with existing methods.
languageDetection(@onNull Bundle results)694         public void languageDetection(@NonNull Bundle results) {
695             try {
696                 mListener.onLanguageDetection(results);
697             } catch (RemoteException e) {
698                 throw e.rethrowFromSystemServer();
699             }
700         }
701 
702         /**
703          * Return the Linux uid assigned to the process that sent you the current transaction that
704          * is being processed. This is obtained from {@link Binder#getCallingUid()}.
705          */
getCallingUid()706         public int getCallingUid() {
707             return mCallingAttributionSource.getUid();
708         }
709 
710         /**
711          * Gets the permission identity of the calling app. If you want to attribute
712          * the mic access to the calling app you can create an attribution context
713          * via {@link android.content.Context#createContext(android.content.ContextParams)}
714          * and passing this identity to {@link
715          * android.content.ContextParams.Builder#setNextAttributionSource(AttributionSource)}.
716          *
717          * @return The permission identity of the calling app.
718          *
719          * @see android.content.ContextParams.Builder#setNextAttributionSource(
720          * AttributionSource)
721          */
722         @SuppressLint("CallbackMethodName")
723         @NonNull
getCallingAttributionSource()724         public AttributionSource getCallingAttributionSource() {
725             return mCallingAttributionSource;
726         }
727 
getAttributionContextForCaller()728         @NonNull Context getAttributionContextForCaller() {
729             if (mAttributionContext == null) {
730                 mAttributionContext = createContext(new ContextParams.Builder()
731                         .setNextAttributionSource(mCallingAttributionSource)
732                         .build());
733             }
734             return mAttributionContext;
735         }
736     }
737 
738     /**
739      * This class receives callbacks from the speech recognition service and forwards them to the
740      * user. An instance of this class is passed to the
741      * {@link RecognitionService#onCheckRecognitionSupport(Intent, SupportCallback)} method. Recognizers may call
742      * these methods on any thread.
743      */
744     public static class SupportCallback {
745         private final IRecognitionSupportCallback mCallback;
746 
SupportCallback( IRecognitionSupportCallback callback)747         private SupportCallback(
748                 IRecognitionSupportCallback callback) {
749             this.mCallback = callback;
750         }
751 
752         /** The service should call this method to notify the caller about the level of support. */
onSupportResult(@onNull RecognitionSupport recognitionSupport)753         public void onSupportResult(@NonNull RecognitionSupport recognitionSupport) {
754             try {
755                 mCallback.onSupportResult(recognitionSupport);
756             } catch (RemoteException e) {
757                 throw e.rethrowFromSystemServer();
758             }
759         }
760 
761         /**
762          * The service should call this method when an error occurred and can't satisfy the support
763          * request.
764          *
765          * @param errorCode code is defined in {@link SpeechRecognizer}
766          */
onError(@peechRecognizer.RecognitionError int errorCode)767         public void onError(@SpeechRecognizer.RecognitionError int errorCode) {
768             try {
769                 mCallback.onError(errorCode);
770             } catch (RemoteException e) {
771                 throw e.rethrowFromSystemServer();
772             }
773         }
774     }
775 
776     /** Binder of the recognition service. */
777     private static final class RecognitionServiceBinder extends IRecognitionService.Stub {
778         private final WeakReference<RecognitionService> mServiceRef;
779 
RecognitionServiceBinder(RecognitionService service)780         public RecognitionServiceBinder(RecognitionService service) {
781             mServiceRef = new WeakReference<>(service);
782         }
783 
784         @Override
startListening(Intent recognizerIntent, IRecognitionListener listener, @NonNull AttributionSource attributionSource)785         public void startListening(Intent recognizerIntent, IRecognitionListener listener,
786                 @NonNull AttributionSource attributionSource) {
787             Objects.requireNonNull(attributionSource);
788             attributionSource.enforceCallingUid();
789             if (DBG) Log.d(TAG, "startListening called by:" + listener.asBinder());
790             final RecognitionService service = mServiceRef.get();
791             if (service != null) {
792                 service.mHandler.sendMessage(Message.obtain(service.mHandler,
793                         MSG_START_LISTENING, new StartListeningArgs(
794                                 recognizerIntent, listener, attributionSource)));
795             }
796         }
797 
798         @Override
stopListening(IRecognitionListener listener)799         public void stopListening(IRecognitionListener listener) {
800             if (DBG) Log.d(TAG, "stopListening called by:" + listener.asBinder());
801             final RecognitionService service = mServiceRef.get();
802             if (service != null) {
803                 service.mHandler.sendMessage(
804                         Message.obtain(service.mHandler, MSG_STOP_LISTENING, listener));
805             }
806         }
807 
808         @Override
cancel(IRecognitionListener listener, boolean isShutdown)809         public void cancel(IRecognitionListener listener, boolean isShutdown) {
810             if (DBG) Log.d(TAG, "cancel called by:" + listener.asBinder());
811             final RecognitionService service = mServiceRef.get();
812             if (service != null) {
813                 service.mHandler.sendMessage(
814                         Message.obtain(service.mHandler, MSG_CANCEL, listener));
815             }
816         }
817 
818         @Override
checkRecognitionSupport( Intent recognizerIntent, @NonNull AttributionSource attributionSource, IRecognitionSupportCallback callback)819         public void checkRecognitionSupport(
820                 Intent recognizerIntent,
821                 @NonNull AttributionSource attributionSource,
822                 IRecognitionSupportCallback callback) {
823             final RecognitionService service = mServiceRef.get();
824             if (service != null) {
825                 service.mHandler.sendMessage(
826                         Message.obtain(service.mHandler, MSG_CHECK_RECOGNITION_SUPPORT,
827                                 new CheckRecognitionSupportArgs(
828                                         recognizerIntent, callback, attributionSource)));
829             }
830         }
831 
832         @Override
triggerModelDownload( Intent recognizerIntent, @NonNull AttributionSource attributionSource, IModelDownloadListener listener)833         public void triggerModelDownload(
834                 Intent recognizerIntent,
835                 @NonNull AttributionSource attributionSource,
836                 IModelDownloadListener listener) {
837             final RecognitionService service = mServiceRef.get();
838             if (service != null) {
839                 service.mHandler.sendMessage(
840                         Message.obtain(
841                                 service.mHandler, MSG_TRIGGER_MODEL_DOWNLOAD,
842                                 new ModelDownloadArgs(
843                                         recognizerIntent,
844                                         attributionSource,
845                                         listener)));
846             }
847         }
848 
clearReference()849         public void clearReference() {
850             mServiceRef.clear();
851         }
852     }
853 
checkPermissionAndStartDataDelivery(SessionState sessionState)854     private boolean checkPermissionAndStartDataDelivery(SessionState sessionState) {
855         if (sessionState.mCallback.mAttributionContextCreated) {
856             return true;
857         }
858 
859         if (PermissionChecker.checkPermissionAndStartDataDelivery(
860                 RecognitionService.this,
861                 Manifest.permission.RECORD_AUDIO,
862                 sessionState.mCallback.getAttributionContextForCaller().getAttributionSource(),
863                 /* message */ null)
864                 == PermissionChecker.PERMISSION_GRANTED) {
865             sessionState.mStartedDataDelivery = true;
866         }
867 
868         return sessionState.mStartedDataDelivery;
869     }
870 
checkPermissionForPreflightNotHardDenied(AttributionSource attributionSource)871     private boolean checkPermissionForPreflightNotHardDenied(AttributionSource attributionSource) {
872         int result = PermissionChecker.checkPermissionForPreflight(RecognitionService.this,
873                 Manifest.permission.RECORD_AUDIO, attributionSource);
874         return result == PermissionChecker.PERMISSION_GRANTED
875                 || result == PermissionChecker.PERMISSION_SOFT_DENIED;
876     }
877 
finishDataDelivery(SessionState sessionState)878     void finishDataDelivery(SessionState sessionState) {
879         if (sessionState.mStartedDataDelivery) {
880             sessionState.mStartedDataDelivery = false;
881             final String op = AppOpsManager.permissionToOp(Manifest.permission.RECORD_AUDIO);
882             PermissionChecker.finishDataDelivery(RecognitionService.this, op,
883                     sessionState.mCallback.getAttributionContextForCaller().getAttributionSource());
884         }
885     }
886 
887     /**
888      * Data class containing information about an ongoing session:
889      * <ul>
890      *   <li> {@link SessionState#mCallback} - callback of the client that invoked the
891      *   {@link RecognitionService#onStartListening(Intent, Callback)} method;
892      *   <li> {@link SessionState#mStartedDataDelivery} - flag denoting if data
893      *   is being delivered to the client.
894      */
895     private static class SessionState {
896         private Callback mCallback;
897         private boolean mStartedDataDelivery;
898 
SessionState(Callback callback, boolean startedDataDelivery)899         SessionState(Callback callback, boolean startedDataDelivery) {
900             mCallback = callback;
901             mStartedDataDelivery = startedDataDelivery;
902         }
903 
SessionState(Callback currentCallback)904         SessionState(Callback currentCallback) {
905             this(currentCallback, false);
906         }
907 
reset()908         void reset() {
909             mCallback = null;
910             mStartedDataDelivery = false;
911         }
912     }
913 }
914