1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16 17 package android.speech; 18 19 import android.Manifest; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.annotation.SdkConstant; 23 import android.annotation.SdkConstant.SdkConstantType; 24 import android.annotation.SuppressLint; 25 import android.annotation.TestApi; 26 import android.app.AppOpsManager; 27 import android.app.Service; 28 import android.content.AttributionSource; 29 import android.content.Context; 30 import android.content.ContextParams; 31 import android.content.Intent; 32 import android.content.PermissionChecker; 33 import android.os.Binder; 34 import android.os.Bundle; 35 import android.os.Handler; 36 import android.os.IBinder; 37 import android.os.Looper; 38 import android.os.Message; 39 import android.os.RemoteException; 40 import android.util.Log; 41 42 import com.android.internal.annotations.GuardedBy; 43 import com.android.internal.util.function.pooled.PooledLambda; 44 45 import java.lang.ref.WeakReference; 46 import java.util.HashMap; 47 import java.util.Map; 48 import java.util.Objects; 49 50 /** 51 * This class provides a base class for recognition service implementations. This class should be 52 * extended only in case you wish to implement a new speech recognizer. Please note that the 53 * implementation of this service is stateless. 54 */ 55 public abstract class RecognitionService extends Service { 56 /** 57 * The {@link Intent} that must be declared as handled by the service. 58 */ 59 @SdkConstant(SdkConstantType.SERVICE_ACTION) 60 public static final String SERVICE_INTERFACE = "android.speech.RecognitionService"; 61 62 /** 63 * Name under which a RecognitionService component publishes information about itself. 64 * This meta-data should reference an XML resource containing a 65 * <code><{@link android.R.styleable#RecognitionService recognition-service}></code> or 66 * <code><{@link android.R.styleable#RecognitionService on-device-recognition-service} 67 * ></code> tag. 68 */ 69 public static final String SERVICE_META_DATA = "android.speech"; 70 71 /** Log messages identifier */ 72 private static final String TAG = "RecognitionService"; 73 74 /** Debugging flag */ 75 private static final boolean DBG = false; 76 77 private static final int DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT = 1; 78 79 private final Map<IBinder, SessionState> mSessions = new HashMap<>(); 80 81 /** Binder of the recognition service */ 82 private final RecognitionServiceBinder mBinder = new RecognitionServiceBinder(this); 83 84 private static final int MSG_START_LISTENING = 1; 85 86 private static final int MSG_STOP_LISTENING = 2; 87 88 private static final int MSG_CANCEL = 3; 89 90 private static final int MSG_RESET = 4; 91 92 private static final int MSG_CHECK_RECOGNITION_SUPPORT = 5; 93 94 private static final int MSG_TRIGGER_MODEL_DOWNLOAD = 6; 95 96 private final Handler mHandler = new Handler() { 97 @Override 98 public void handleMessage(Message msg) { 99 switch (msg.what) { 100 case MSG_START_LISTENING: 101 StartListeningArgs args = (StartListeningArgs) msg.obj; 102 dispatchStartListening(args.mIntent, args.mListener, args.mAttributionSource); 103 break; 104 case MSG_STOP_LISTENING: 105 dispatchStopListening((IRecognitionListener) msg.obj); 106 break; 107 case MSG_CANCEL: 108 dispatchCancel((IRecognitionListener) msg.obj); 109 break; 110 case MSG_RESET: 111 dispatchClearCallback((IRecognitionListener) msg.obj); 112 break; 113 case MSG_CHECK_RECOGNITION_SUPPORT: 114 CheckRecognitionSupportArgs checkArgs = (CheckRecognitionSupportArgs) msg.obj; 115 dispatchCheckRecognitionSupport( 116 checkArgs.mIntent, checkArgs.callback, checkArgs.mAttributionSource); 117 break; 118 case MSG_TRIGGER_MODEL_DOWNLOAD: 119 ModelDownloadArgs modelDownloadArgs = (ModelDownloadArgs) msg.obj; 120 dispatchTriggerModelDownload( 121 modelDownloadArgs.mIntent, 122 modelDownloadArgs.mAttributionSource, 123 modelDownloadArgs.mListener); 124 break; 125 } 126 } 127 }; 128 dispatchStartListening(Intent intent, final IRecognitionListener listener, @NonNull AttributionSource attributionSource)129 private void dispatchStartListening(Intent intent, final IRecognitionListener listener, 130 @NonNull AttributionSource attributionSource) { 131 Callback currentCallback = null; 132 SessionState sessionState = mSessions.get(listener.asBinder()); 133 134 try { 135 if (sessionState == null) { 136 if (mSessions.size() >= getMaxConcurrentSessionsCount()) { 137 listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); 138 Log.i(TAG, "#startListening received " 139 + "when the service's capacity is full - ignoring this call."); 140 return; 141 } 142 143 boolean preflightPermissionCheckPassed = 144 intent.hasExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE) 145 || checkPermissionForPreflightNotHardDenied(attributionSource); 146 if (preflightPermissionCheckPassed) { 147 currentCallback = new Callback(listener, attributionSource); 148 sessionState = new SessionState(currentCallback); 149 mSessions.put(listener.asBinder(), sessionState); 150 if (DBG) { 151 Log.d(TAG, "Added a new session to the map, pending permission checks"); 152 } 153 RecognitionService.this.onStartListening(intent, currentCallback); 154 } 155 156 if (!preflightPermissionCheckPassed 157 || !checkPermissionAndStartDataDelivery(sessionState)) { 158 listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS); 159 if (preflightPermissionCheckPassed) { 160 // If start listening was attempted, cancel the callback. 161 RecognitionService.this.onCancel(currentCallback); 162 mSessions.remove(listener.asBinder()); 163 finishDataDelivery(sessionState); 164 sessionState.reset(); 165 } 166 Log.i(TAG, "#startListening received from a caller " 167 + "without permission " + Manifest.permission.RECORD_AUDIO + "."); 168 } 169 } else { 170 listener.onError(SpeechRecognizer.ERROR_CLIENT); 171 Log.i(TAG, "#startListening received " 172 + "for a listener which is already in session - ignoring this call."); 173 } 174 } catch (RemoteException e) { 175 Log.d(TAG, "#onError call from #startListening failed."); 176 } 177 } 178 dispatchStopListening(IRecognitionListener listener)179 private void dispatchStopListening(IRecognitionListener listener) { 180 SessionState sessionState = mSessions.get(listener.asBinder()); 181 if (sessionState == null) { 182 try { 183 listener.onError(SpeechRecognizer.ERROR_CLIENT); 184 } catch (RemoteException e) { 185 Log.d(TAG, "#onError call from #stopListening failed."); 186 } 187 Log.w(TAG, "#stopListening received for a listener " 188 + "which has not started a session - ignoring this call."); 189 } else { 190 RecognitionService.this.onStopListening(sessionState.mCallback); 191 } 192 } 193 dispatchCancel(IRecognitionListener listener)194 private void dispatchCancel(IRecognitionListener listener) { 195 SessionState sessionState = mSessions.get(listener.asBinder()); 196 if (sessionState == null) { 197 Log.w(TAG, "#cancel received for a listener which has not started a session " 198 + "- ignoring this call."); 199 } else { 200 RecognitionService.this.onCancel(sessionState.mCallback); 201 dispatchClearCallback(listener); 202 } 203 } 204 dispatchClearCallback(IRecognitionListener listener)205 private void dispatchClearCallback(IRecognitionListener listener) { 206 SessionState sessionState = mSessions.remove(listener.asBinder()); 207 if (sessionState != null) { 208 if (DBG) { 209 Log.d(TAG, "Removed session from the map for listener = " 210 + listener.asBinder() + "."); 211 } 212 finishDataDelivery(sessionState); 213 sessionState.reset(); 214 } 215 } 216 dispatchCheckRecognitionSupport( Intent intent, IRecognitionSupportCallback callback, AttributionSource attributionSource)217 private void dispatchCheckRecognitionSupport( 218 Intent intent, IRecognitionSupportCallback callback, 219 AttributionSource attributionSource) { 220 RecognitionService.this.onCheckRecognitionSupport( 221 intent, 222 attributionSource, 223 new SupportCallback(callback)); 224 } 225 dispatchTriggerModelDownload( Intent intent, AttributionSource attributionSource, IModelDownloadListener listener)226 private void dispatchTriggerModelDownload( 227 Intent intent, 228 AttributionSource attributionSource, 229 IModelDownloadListener listener) { 230 if (listener == null) { 231 RecognitionService.this.onTriggerModelDownload(intent, attributionSource); 232 } else { 233 RecognitionService.this.onTriggerModelDownload( 234 intent, 235 attributionSource, 236 new ModelDownloadListener() { 237 238 private final Object mLock = new Object(); 239 240 @GuardedBy("mLock") 241 private boolean mIsTerminated = false; 242 243 @Override 244 public void onProgress(int completedPercent) { 245 synchronized (mLock) { 246 if (mIsTerminated) { 247 return; 248 } 249 try { 250 listener.onProgress(completedPercent); 251 } catch (RemoteException e) { 252 throw e.rethrowFromSystemServer(); 253 } 254 } 255 } 256 257 @Override 258 public void onSuccess() { 259 synchronized (mLock) { 260 if (mIsTerminated) { 261 return; 262 } 263 mIsTerminated = true; 264 try { 265 listener.onSuccess(); 266 } catch (RemoteException e) { 267 throw e.rethrowFromSystemServer(); 268 } 269 } 270 } 271 272 @Override 273 public void onScheduled() { 274 synchronized (mLock) { 275 if (mIsTerminated) { 276 return; 277 } 278 mIsTerminated = true; 279 try { 280 listener.onScheduled(); 281 } catch (RemoteException e) { 282 throw e.rethrowFromSystemServer(); 283 } 284 } 285 } 286 287 @Override 288 public void onError(int error) { 289 synchronized (mLock) { 290 if (mIsTerminated) { 291 return; 292 } 293 mIsTerminated = true; 294 try { 295 listener.onError(error); 296 } catch (RemoteException e) { 297 throw e.rethrowFromSystemServer(); 298 } 299 } 300 } 301 }); 302 } 303 } 304 305 private static class StartListeningArgs { 306 public final Intent mIntent; 307 308 public final IRecognitionListener mListener; 309 @NonNull public final AttributionSource mAttributionSource; 310 StartListeningArgs(Intent intent, IRecognitionListener listener, @NonNull AttributionSource attributionSource)311 public StartListeningArgs(Intent intent, IRecognitionListener listener, 312 @NonNull AttributionSource attributionSource) { 313 this.mIntent = intent; 314 this.mListener = listener; 315 this.mAttributionSource = attributionSource; 316 } 317 } 318 319 private static class CheckRecognitionSupportArgs { 320 public final Intent mIntent; 321 public final IRecognitionSupportCallback callback; 322 public final AttributionSource mAttributionSource; 323 CheckRecognitionSupportArgs( Intent intent, IRecognitionSupportCallback callback, AttributionSource attributionSource)324 private CheckRecognitionSupportArgs( 325 Intent intent, 326 IRecognitionSupportCallback callback, 327 AttributionSource attributionSource) { 328 this.mIntent = intent; 329 this.callback = callback; 330 this.mAttributionSource = attributionSource; 331 } 332 } 333 334 private static class ModelDownloadArgs { 335 final Intent mIntent; 336 final AttributionSource mAttributionSource; 337 @Nullable final IModelDownloadListener mListener; 338 ModelDownloadArgs( Intent intent, AttributionSource attributionSource, @Nullable IModelDownloadListener listener)339 private ModelDownloadArgs( 340 Intent intent, 341 AttributionSource attributionSource, 342 @Nullable IModelDownloadListener listener) { 343 this.mIntent = intent; 344 this.mAttributionSource = attributionSource; 345 this.mListener = listener; 346 } 347 } 348 349 /** 350 * Notifies the service that it should start listening for speech. 351 * 352 * <p> If you are recognizing speech from the microphone, in this callback you 353 * should create an attribution context for the caller such that when you access 354 * the mic the caller would be properly blamed (and their permission checked in 355 * the process) for accessing the microphone and that you served as a proxy for 356 * this sensitive data (and your permissions would be checked in the process). 357 * You should also open the mic in this callback via the attribution context 358 * and close the mic before returning the recognized result. If you don't do 359 * that then the caller would be blamed and you as being a proxy as well as you 360 * would get one more blame on yourself when you open the microphone. 361 * 362 * <pre> 363 * Context attributionContext = context.createContext(new ContextParams.Builder() 364 * .setNextAttributionSource(callback.getCallingAttributionSource()) 365 * .build()); 366 * 367 * AudioRecord recorder = AudioRecord.Builder() 368 * .setContext(attributionContext); 369 * . . . 370 * .build(); 371 * 372 * recorder.startRecording() 373 * </pre> 374 * 375 * @param recognizerIntent contains parameters for the recognition to be performed. The intent 376 * may also contain optional extras, see {@link RecognizerIntent}. If these values are 377 * not set explicitly, default values should be used by the recognizer. 378 * @param listener that will receive the service's callbacks 379 */ onStartListening(Intent recognizerIntent, Callback listener)380 protected abstract void onStartListening(Intent recognizerIntent, Callback listener); 381 382 /** 383 * Notifies the service that it should cancel the speech recognition. 384 */ onCancel(Callback listener)385 protected abstract void onCancel(Callback listener); 386 387 /** 388 * Notifies the service that it should stop listening for speech. Speech captured so far should 389 * be recognized as if the user had stopped speaking at this point. This method is only called 390 * if the application calls it explicitly. 391 */ onStopListening(Callback listener)392 protected abstract void onStopListening(Callback listener); 393 394 /** 395 * Queries the service on whether it would support a {@link #onStartListening(Intent, Callback)} 396 * for the same {@code recognizerIntent}. 397 * 398 * <p>The service will notify the caller about the level of support or error via 399 * {@link SupportCallback}. 400 * 401 * <p>If the service does not offer the support check it will notify the caller with 402 * {@link SpeechRecognizer#ERROR_CANNOT_CHECK_SUPPORT}. 403 */ onCheckRecognitionSupport( @onNull Intent recognizerIntent, @NonNull SupportCallback supportCallback)404 public void onCheckRecognitionSupport( 405 @NonNull Intent recognizerIntent, 406 @NonNull SupportCallback supportCallback) { 407 if (DBG) { 408 Log.i(TAG, String.format("#onSupports [%s]", recognizerIntent)); 409 } 410 supportCallback.onError(SpeechRecognizer.ERROR_CANNOT_CHECK_SUPPORT); 411 } 412 413 /** 414 * Queries the service on whether it would support a {@link #onStartListening(Intent, Callback)} 415 * for the same {@code recognizerIntent}. 416 * 417 * <p>The service will notify the caller about the level of support or error via 418 * {@link SupportCallback}. 419 * 420 * <p>If the service does not offer the support check it will notify the caller with 421 * {@link SpeechRecognizer#ERROR_CANNOT_CHECK_SUPPORT}. 422 * 423 * <p>Provides the calling AttributionSource to the service implementation so that permissions 424 * and bandwidth could be correctly blamed.</p> 425 */ onCheckRecognitionSupport( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource, @NonNull SupportCallback supportCallback)426 public void onCheckRecognitionSupport( 427 @NonNull Intent recognizerIntent, 428 @NonNull AttributionSource attributionSource, 429 @NonNull SupportCallback supportCallback) { 430 onCheckRecognitionSupport(recognizerIntent, supportCallback); 431 } 432 433 /** 434 * Requests the download of the recognizer support for {@code recognizerIntent}. 435 */ onTriggerModelDownload(@onNull Intent recognizerIntent)436 public void onTriggerModelDownload(@NonNull Intent recognizerIntent) { 437 if (DBG) { 438 Log.i(TAG, String.format("#downloadModel [%s]", recognizerIntent)); 439 } 440 } 441 442 /** 443 * Requests the download of the recognizer support for {@code recognizerIntent}. 444 * 445 * <p>Provides the calling AttributionSource to the service implementation so that permissions 446 * and bandwidth could be correctly blamed.</p> 447 */ onTriggerModelDownload( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource)448 public void onTriggerModelDownload( 449 @NonNull Intent recognizerIntent, 450 @NonNull AttributionSource attributionSource) { 451 onTriggerModelDownload(recognizerIntent); 452 } 453 454 /** 455 * Requests the download of the recognizer support for {@code recognizerIntent}. 456 * 457 * <p> Provides the calling {@link AttributionSource} to the service implementation so that 458 * permissions and bandwidth could be correctly blamed. 459 * 460 * <p> Client will receive the progress updates via the given {@link ModelDownloadListener}: 461 * 462 * <li> If the model is already available, {@link ModelDownloadListener#onSuccess()} will be 463 * called directly. The model can be safely used afterwards. 464 * 465 * <li> If the {@link RecognitionService} has started the download, 466 * {@link ModelDownloadListener#onProgress(int)} will be called an unspecified (zero or more) 467 * number of times until the download is complete. 468 * When the download finishes, {@link ModelDownloadListener#onSuccess()} will be called. 469 * The model can be safely used afterwards. 470 * 471 * <li> If the {@link RecognitionService} has only scheduled the download, but won't satisfy it 472 * immediately, {@link ModelDownloadListener#onScheduled()} will be called. 473 * There will be no further updates on this listener. 474 * 475 * <li> If the request fails at any time due to a network or scheduling error, 476 * {@link ModelDownloadListener#onError(int)} will be called. 477 * 478 * @param recognizerIntent contains parameters for the recognition to be performed. The intent 479 * may also contain optional extras, see {@link RecognizerIntent}. 480 * @param attributionSource the attribution source of the caller. 481 * @param listener on which to receive updates about the model download request. 482 */ onTriggerModelDownload( @onNull Intent recognizerIntent, @NonNull AttributionSource attributionSource, @NonNull ModelDownloadListener listener)483 public void onTriggerModelDownload( 484 @NonNull Intent recognizerIntent, 485 @NonNull AttributionSource attributionSource, 486 @NonNull ModelDownloadListener listener) { 487 listener.onError(SpeechRecognizer.ERROR_CANNOT_LISTEN_TO_DOWNLOAD_EVENTS); 488 } 489 490 @Override 491 @SuppressLint("MissingNullability") createContext(@onNull ContextParams contextParams)492 public Context createContext(@NonNull ContextParams contextParams) { 493 if (contextParams.getNextAttributionSource() != null) { 494 if (mHandler.getLooper().equals(Looper.myLooper())) { 495 handleAttributionContextCreation(contextParams.getNextAttributionSource()); 496 } else { 497 mHandler.sendMessage( 498 PooledLambda.obtainMessage(this::handleAttributionContextCreation, 499 contextParams.getNextAttributionSource())); 500 } 501 } 502 return super.createContext(contextParams); 503 } 504 handleAttributionContextCreation(@onNull AttributionSource attributionSource)505 private void handleAttributionContextCreation(@NonNull AttributionSource attributionSource) { 506 for (SessionState sessionState : mSessions.values()) { 507 Callback currentCallback = sessionState.mCallback; 508 if (currentCallback != null 509 && currentCallback.mCallingAttributionSource.equals(attributionSource)) { 510 currentCallback.mAttributionContextCreated = true; 511 } 512 } 513 } 514 515 @Override onBind(final Intent intent)516 public final IBinder onBind(final Intent intent) { 517 if (DBG) Log.d(TAG, "#onBind, intent=" + intent); 518 onBindInternal(); 519 return mBinder; 520 } 521 522 /** @hide */ 523 @SuppressLint("UnflaggedApi") // @TestApi without associated feature. 524 @TestApi onBindInternal()525 public void onBindInternal() { } 526 527 @Override onDestroy()528 public void onDestroy() { 529 if (DBG) Log.d(TAG, "#onDestroy"); 530 for (SessionState sessionState : mSessions.values()) { 531 finishDataDelivery(sessionState); 532 sessionState.reset(); 533 } 534 mSessions.clear(); 535 mBinder.clearReference(); 536 super.onDestroy(); 537 } 538 539 /** 540 * Returns the maximal number of recognition sessions ongoing at the same time. 541 * <p> 542 * The default value is 1, meaning concurrency should be enabled by overriding this method. 543 */ getMaxConcurrentSessionsCount()544 public int getMaxConcurrentSessionsCount() { 545 return DEFAULT_MAX_CONCURRENT_SESSIONS_COUNT; 546 } 547 548 /** 549 * This class receives callbacks from the speech recognition service and forwards them to the 550 * user. An instance of this class is passed to the 551 * {@link RecognitionService#onStartListening(Intent, Callback)} method. Recognizers may call 552 * these methods on any thread. 553 */ 554 public class Callback { 555 private final IRecognitionListener mListener; 556 @NonNull private final AttributionSource mCallingAttributionSource; 557 @Nullable private Context mAttributionContext; 558 private boolean mAttributionContextCreated; 559 Callback(IRecognitionListener listener, @NonNull AttributionSource attributionSource)560 private Callback(IRecognitionListener listener, 561 @NonNull AttributionSource attributionSource) { 562 mListener = listener; 563 mCallingAttributionSource = attributionSource; 564 } 565 566 /** 567 * The service should call this method when the user has started to speak. 568 */ beginningOfSpeech()569 public void beginningOfSpeech() throws RemoteException { 570 mListener.onBeginningOfSpeech(); 571 } 572 573 /** 574 * The service should call this method when sound has been received. The purpose of this 575 * function is to allow giving feedback to the user regarding the captured audio. 576 * 577 * @param buffer a buffer containing a sequence of big-endian 16-bit integers representing a 578 * single channel audio stream. The sample rate is implementation dependent. 579 */ bufferReceived(byte[] buffer)580 public void bufferReceived(byte[] buffer) throws RemoteException { 581 mListener.onBufferReceived(buffer); 582 } 583 584 /** 585 * The service should call this method after the user stops speaking. 586 */ endOfSpeech()587 public void endOfSpeech() throws RemoteException { 588 mListener.onEndOfSpeech(); 589 } 590 591 /** 592 * The service should call this method when a network or recognition error occurred. 593 * 594 * @param error code is defined in {@link SpeechRecognizer} 595 */ error(@peechRecognizer.RecognitionError int error)596 public void error(@SpeechRecognizer.RecognitionError int error) throws RemoteException { 597 Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget(); 598 mListener.onError(error); 599 } 600 601 /** 602 * The service should call this method when partial recognition results are available. This 603 * method can be called at any time between {@link #beginningOfSpeech()} and 604 * {@link #results(Bundle)} when partial results are ready. This method may be called zero, 605 * one or multiple times for each call to {@link SpeechRecognizer#startListening(Intent)}, 606 * depending on the speech recognition service implementation. 607 * 608 * @param partialResults the returned results. To retrieve the results in 609 * ArrayList<String> format use {@link Bundle#getStringArrayList(String)} with 610 * {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter 611 */ partialResults(Bundle partialResults)612 public void partialResults(Bundle partialResults) throws RemoteException { 613 mListener.onPartialResults(partialResults); 614 } 615 616 /** 617 * The service should call this method when the endpointer is ready for the user to start 618 * speaking. 619 * 620 * @param params parameters set by the recognition service. Reserved for future use. 621 */ readyForSpeech(Bundle params)622 public void readyForSpeech(Bundle params) throws RemoteException { 623 mListener.onReadyForSpeech(params); 624 } 625 626 /** 627 * The service should call this method when recognition results are ready. 628 * 629 * @param results the recognition results. To retrieve the results in {@code 630 * ArrayList<String>} format use {@link Bundle#getStringArrayList(String)} with 631 * {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter 632 */ results(Bundle results)633 public void results(Bundle results) throws RemoteException { 634 Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget(); 635 mListener.onResults(results); 636 } 637 638 /** 639 * The service should call this method when the sound level in the audio stream has changed. 640 * There is no guarantee that this method will be called. 641 * 642 * @param rmsdB the new RMS dB value 643 */ rmsChanged(float rmsdB)644 public void rmsChanged(float rmsdB) throws RemoteException { 645 mListener.onRmsChanged(rmsdB); 646 } 647 648 /** 649 * The service should call this method for each ready segment of a long recognition session. 650 * 651 * @param results the recognition results. To retrieve the results in {@code 652 * ArrayList<String>} format use {@link Bundle#getStringArrayList(String)} with 653 * {@link SpeechRecognizer#RESULTS_RECOGNITION} as a parameter 654 */ 655 @SuppressLint({"CallbackMethodName", "RethrowRemoteException"}) segmentResults(@onNull Bundle results)656 public void segmentResults(@NonNull Bundle results) throws RemoteException { 657 mListener.onSegmentResults(results); 658 } 659 660 /** 661 * The service should call this method to end a segmented session. 662 */ 663 @SuppressLint({"CallbackMethodName", "RethrowRemoteException"}) endOfSegmentedSession()664 public void endOfSegmentedSession() throws RemoteException { 665 Message.obtain(mHandler, MSG_RESET, mListener).sendToTarget(); 666 mListener.onEndOfSegmentedSession(); 667 } 668 669 /** 670 * The service should call this method when the language detection (and switching) 671 * results are available. This method can be called on any number of occasions 672 * at any time between {@link #beginningOfSpeech()} and {@link #endOfSpeech()}, 673 * depending on the speech recognition service implementation. 674 * 675 * @param results the returned language detection (and switching) results. 676 * <p> To retrieve the most confidently detected language IETF tag 677 * (as defined by BCP 47, e.g., "en-US", "de-DE"), 678 * use {@link Bundle#getString(String)} 679 * with {@link SpeechRecognizer#DETECTED_LANGUAGE} as the parameter. 680 * <p> To retrieve the language detection confidence level represented by a value 681 * prefixed by {@code LANGUAGE_DETECTION_CONFIDENCE_LEVEL_} defined in 682 * {@link SpeechRecognizer}, use {@link Bundle#getInt(String)} with 683 * {@link SpeechRecognizer#LANGUAGE_DETECTION_CONFIDENCE_LEVEL} as the parameter. 684 * <p> To retrieve the alternative locales for the same language 685 * retrieved by the key {@link SpeechRecognizer#DETECTED_LANGUAGE}, 686 * use {@link Bundle#getStringArrayList(String)} 687 * with {@link SpeechRecognizer#TOP_LOCALE_ALTERNATIVES} as the parameter. 688 * <p> To retrieve the language switching results represented by a value 689 * prefixed by {@code LANGUAGE_SWITCH_RESULT_} 690 * and defined in {@link SpeechRecognizer}, use {@link Bundle#getInt(String)} 691 * with {@link SpeechRecognizer#LANGUAGE_SWITCH_RESULT} as the parameter. 692 */ 693 @SuppressLint("CallbackMethodName") // For consistency with existing methods. languageDetection(@onNull Bundle results)694 public void languageDetection(@NonNull Bundle results) { 695 try { 696 mListener.onLanguageDetection(results); 697 } catch (RemoteException e) { 698 throw e.rethrowFromSystemServer(); 699 } 700 } 701 702 /** 703 * Return the Linux uid assigned to the process that sent you the current transaction that 704 * is being processed. This is obtained from {@link Binder#getCallingUid()}. 705 */ getCallingUid()706 public int getCallingUid() { 707 return mCallingAttributionSource.getUid(); 708 } 709 710 /** 711 * Gets the permission identity of the calling app. If you want to attribute 712 * the mic access to the calling app you can create an attribution context 713 * via {@link android.content.Context#createContext(android.content.ContextParams)} 714 * and passing this identity to {@link 715 * android.content.ContextParams.Builder#setNextAttributionSource(AttributionSource)}. 716 * 717 * @return The permission identity of the calling app. 718 * 719 * @see android.content.ContextParams.Builder#setNextAttributionSource( 720 * AttributionSource) 721 */ 722 @SuppressLint("CallbackMethodName") 723 @NonNull getCallingAttributionSource()724 public AttributionSource getCallingAttributionSource() { 725 return mCallingAttributionSource; 726 } 727 getAttributionContextForCaller()728 @NonNull Context getAttributionContextForCaller() { 729 if (mAttributionContext == null) { 730 mAttributionContext = createContext(new ContextParams.Builder() 731 .setNextAttributionSource(mCallingAttributionSource) 732 .build()); 733 } 734 return mAttributionContext; 735 } 736 } 737 738 /** 739 * This class receives callbacks from the speech recognition service and forwards them to the 740 * user. An instance of this class is passed to the 741 * {@link RecognitionService#onCheckRecognitionSupport(Intent, SupportCallback)} method. Recognizers may call 742 * these methods on any thread. 743 */ 744 public static class SupportCallback { 745 private final IRecognitionSupportCallback mCallback; 746 SupportCallback( IRecognitionSupportCallback callback)747 private SupportCallback( 748 IRecognitionSupportCallback callback) { 749 this.mCallback = callback; 750 } 751 752 /** The service should call this method to notify the caller about the level of support. */ onSupportResult(@onNull RecognitionSupport recognitionSupport)753 public void onSupportResult(@NonNull RecognitionSupport recognitionSupport) { 754 try { 755 mCallback.onSupportResult(recognitionSupport); 756 } catch (RemoteException e) { 757 throw e.rethrowFromSystemServer(); 758 } 759 } 760 761 /** 762 * The service should call this method when an error occurred and can't satisfy the support 763 * request. 764 * 765 * @param errorCode code is defined in {@link SpeechRecognizer} 766 */ onError(@peechRecognizer.RecognitionError int errorCode)767 public void onError(@SpeechRecognizer.RecognitionError int errorCode) { 768 try { 769 mCallback.onError(errorCode); 770 } catch (RemoteException e) { 771 throw e.rethrowFromSystemServer(); 772 } 773 } 774 } 775 776 /** Binder of the recognition service. */ 777 private static final class RecognitionServiceBinder extends IRecognitionService.Stub { 778 private final WeakReference<RecognitionService> mServiceRef; 779 RecognitionServiceBinder(RecognitionService service)780 public RecognitionServiceBinder(RecognitionService service) { 781 mServiceRef = new WeakReference<>(service); 782 } 783 784 @Override startListening(Intent recognizerIntent, IRecognitionListener listener, @NonNull AttributionSource attributionSource)785 public void startListening(Intent recognizerIntent, IRecognitionListener listener, 786 @NonNull AttributionSource attributionSource) { 787 Objects.requireNonNull(attributionSource); 788 attributionSource.enforceCallingUid(); 789 if (DBG) Log.d(TAG, "startListening called by:" + listener.asBinder()); 790 final RecognitionService service = mServiceRef.get(); 791 if (service != null) { 792 service.mHandler.sendMessage(Message.obtain(service.mHandler, 793 MSG_START_LISTENING, new StartListeningArgs( 794 recognizerIntent, listener, attributionSource))); 795 } 796 } 797 798 @Override stopListening(IRecognitionListener listener)799 public void stopListening(IRecognitionListener listener) { 800 if (DBG) Log.d(TAG, "stopListening called by:" + listener.asBinder()); 801 final RecognitionService service = mServiceRef.get(); 802 if (service != null) { 803 service.mHandler.sendMessage( 804 Message.obtain(service.mHandler, MSG_STOP_LISTENING, listener)); 805 } 806 } 807 808 @Override cancel(IRecognitionListener listener, boolean isShutdown)809 public void cancel(IRecognitionListener listener, boolean isShutdown) { 810 if (DBG) Log.d(TAG, "cancel called by:" + listener.asBinder()); 811 final RecognitionService service = mServiceRef.get(); 812 if (service != null) { 813 service.mHandler.sendMessage( 814 Message.obtain(service.mHandler, MSG_CANCEL, listener)); 815 } 816 } 817 818 @Override checkRecognitionSupport( Intent recognizerIntent, @NonNull AttributionSource attributionSource, IRecognitionSupportCallback callback)819 public void checkRecognitionSupport( 820 Intent recognizerIntent, 821 @NonNull AttributionSource attributionSource, 822 IRecognitionSupportCallback callback) { 823 final RecognitionService service = mServiceRef.get(); 824 if (service != null) { 825 service.mHandler.sendMessage( 826 Message.obtain(service.mHandler, MSG_CHECK_RECOGNITION_SUPPORT, 827 new CheckRecognitionSupportArgs( 828 recognizerIntent, callback, attributionSource))); 829 } 830 } 831 832 @Override triggerModelDownload( Intent recognizerIntent, @NonNull AttributionSource attributionSource, IModelDownloadListener listener)833 public void triggerModelDownload( 834 Intent recognizerIntent, 835 @NonNull AttributionSource attributionSource, 836 IModelDownloadListener listener) { 837 final RecognitionService service = mServiceRef.get(); 838 if (service != null) { 839 service.mHandler.sendMessage( 840 Message.obtain( 841 service.mHandler, MSG_TRIGGER_MODEL_DOWNLOAD, 842 new ModelDownloadArgs( 843 recognizerIntent, 844 attributionSource, 845 listener))); 846 } 847 } 848 clearReference()849 public void clearReference() { 850 mServiceRef.clear(); 851 } 852 } 853 checkPermissionAndStartDataDelivery(SessionState sessionState)854 private boolean checkPermissionAndStartDataDelivery(SessionState sessionState) { 855 if (sessionState.mCallback.mAttributionContextCreated) { 856 return true; 857 } 858 859 if (PermissionChecker.checkPermissionAndStartDataDelivery( 860 RecognitionService.this, 861 Manifest.permission.RECORD_AUDIO, 862 sessionState.mCallback.getAttributionContextForCaller().getAttributionSource(), 863 /* message */ null) 864 == PermissionChecker.PERMISSION_GRANTED) { 865 sessionState.mStartedDataDelivery = true; 866 } 867 868 return sessionState.mStartedDataDelivery; 869 } 870 checkPermissionForPreflightNotHardDenied(AttributionSource attributionSource)871 private boolean checkPermissionForPreflightNotHardDenied(AttributionSource attributionSource) { 872 int result = PermissionChecker.checkPermissionForPreflight(RecognitionService.this, 873 Manifest.permission.RECORD_AUDIO, attributionSource); 874 return result == PermissionChecker.PERMISSION_GRANTED 875 || result == PermissionChecker.PERMISSION_SOFT_DENIED; 876 } 877 finishDataDelivery(SessionState sessionState)878 void finishDataDelivery(SessionState sessionState) { 879 if (sessionState.mStartedDataDelivery) { 880 sessionState.mStartedDataDelivery = false; 881 final String op = AppOpsManager.permissionToOp(Manifest.permission.RECORD_AUDIO); 882 PermissionChecker.finishDataDelivery(RecognitionService.this, op, 883 sessionState.mCallback.getAttributionContextForCaller().getAttributionSource()); 884 } 885 } 886 887 /** 888 * Data class containing information about an ongoing session: 889 * <ul> 890 * <li> {@link SessionState#mCallback} - callback of the client that invoked the 891 * {@link RecognitionService#onStartListening(Intent, Callback)} method; 892 * <li> {@link SessionState#mStartedDataDelivery} - flag denoting if data 893 * is being delivered to the client. 894 */ 895 private static class SessionState { 896 private Callback mCallback; 897 private boolean mStartedDataDelivery; 898 SessionState(Callback callback, boolean startedDataDelivery)899 SessionState(Callback callback, boolean startedDataDelivery) { 900 mCallback = callback; 901 mStartedDataDelivery = startedDataDelivery; 902 } 903 SessionState(Callback currentCallback)904 SessionState(Callback currentCallback) { 905 this(currentCallback, false); 906 } 907 reset()908 void reset() { 909 mCallback = null; 910 mStartedDataDelivery = false; 911 } 912 } 913 } 914