/* * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.speech; import android.Manifest; import android.annotation.IntDef; import android.annotation.MainThread; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.RequiresPermission; import android.annotation.TestApi; import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.pm.ResolveInfo; import android.os.Binder; import android.os.Bundle; import android.os.Handler; import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.os.RemoteException; import android.os.ServiceManager; import android.provider.Settings; import android.text.TextUtils; import android.util.Log; import android.util.Slog; import com.android.internal.R; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.List; import java.util.Queue; import java.util.concurrent.LinkedBlockingQueue; /** * This class provides access to the speech recognition service. This service allows access to the * speech recognizer. Do not instantiate this class directly, instead, call * {@link SpeechRecognizer#createSpeechRecognizer(Context)}, or * {@link SpeechRecognizer#createOnDeviceSpeechRecognizer(Context)}. This class's methods must be * invoked only from the main application thread. * *
The implementation of this API is likely to stream audio to remote servers to perform speech * recognition. As such this API is not intended to be used for continuous recognition, which would * consume a significant amount of battery and bandwidth. * *
Please note that the application must have {@link android.Manifest.permission#RECORD_AUDIO}
* permission to use this class.
*/
public class SpeechRecognizer {
/** DEBUG value to enable verbose debug prints */
private static final boolean DBG = false;
/** Log messages identifier */
private static final String TAG = "SpeechRecognizer";
/**
* Key used to retrieve an {@code ArrayList
* Confidence values close to 1.0 indicate high confidence (the speech recognizer is confident
* that the recognition result is correct), while values close to 0.0 indicate low confidence.
*
* This value is optional and might not be provided.
*/
public static final String CONFIDENCE_SCORES = "confidence_scores";
/**
* The reason speech recognition failed.
*
* @hide
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef(prefix = {"ERROR_"}, value = {
ERROR_NETWORK_TIMEOUT,
ERROR_NETWORK,
ERROR_AUDIO,
ERROR_SERVER,
ERROR_CLIENT,
ERROR_SPEECH_TIMEOUT,
ERROR_NO_MATCH,
ERROR_RECOGNIZER_BUSY,
ERROR_INSUFFICIENT_PERMISSIONS,
ERROR_TOO_MANY_REQUESTS,
ERROR_SERVER_DISCONNECTED,
ERROR_LANGUAGE_NOT_SUPPORTED,
ERROR_LANGUAGE_UNAVAILABLE
})
public @interface RecognitionError {}
/** Network operation timed out. */
public static final int ERROR_NETWORK_TIMEOUT = 1;
/** Other network related errors. */
public static final int ERROR_NETWORK = 2;
/** Audio recording error. */
public static final int ERROR_AUDIO = 3;
/** Server sends error status. */
public static final int ERROR_SERVER = 4;
/** Other client side errors. */
public static final int ERROR_CLIENT = 5;
/** No speech input */
public static final int ERROR_SPEECH_TIMEOUT = 6;
/** No recognition result matched. */
public static final int ERROR_NO_MATCH = 7;
/** RecognitionService busy. */
public static final int ERROR_RECOGNIZER_BUSY = 8;
/** Insufficient permissions */
public static final int ERROR_INSUFFICIENT_PERMISSIONS = 9;
/** Too many requests from the same client. */
public static final int ERROR_TOO_MANY_REQUESTS = 10;
/** Server has been disconnected, e.g. because the app has crashed. */
public static final int ERROR_SERVER_DISCONNECTED = 11;
/** Requested language is not available to be used with the current recognizer. */
public static final int ERROR_LANGUAGE_NOT_SUPPORTED = 12;
/** Requested language is supported, but not available currently (e.g. not downloaded yet). */
public static final int ERROR_LANGUAGE_UNAVAILABLE = 13;
/** action codes */
private static final int MSG_START = 1;
private static final int MSG_STOP = 2;
private static final int MSG_CANCEL = 3;
private static final int MSG_CHANGE_LISTENER = 4;
private static final int MSG_SET_TEMPORARY_ON_DEVICE_COMPONENT = 5;
/** The actual RecognitionService endpoint */
private IRecognitionService mService;
/** Context with which the manager was created */
private final Context mContext;
/** Component to direct service intent to */
private final ComponentName mServiceComponent;
/** Whether to use on-device speech recognizer. */
private final boolean mOnDevice;
private IRecognitionServiceManager mManagerService;
/** Handler that will execute the main tasks */
private Handler mHandler = new Handler(Looper.getMainLooper()) {
@Override
public void handleMessage(Message msg) {
switch (msg.what) {
case MSG_START:
handleStartListening((Intent) msg.obj);
break;
case MSG_STOP:
handleStopMessage();
break;
case MSG_CANCEL:
handleCancelMessage();
break;
case MSG_CHANGE_LISTENER:
handleChangeListener((RecognitionListener) msg.obj);
break;
case MSG_SET_TEMPORARY_ON_DEVICE_COMPONENT:
handleSetTemporaryComponent((ComponentName) msg.obj);
break;
}
}
};
/**
* Temporary queue, saving the messages until the connection will be established, afterwards,
* only mHandler will receive the messages
*/
private final Queue For apps targeting Android 11 (API level 30) interaction with a speech recognition
* service requires Important: before calling this method, please check via
* {@link android.content.pm.PackageManager#queryIntentServices(Intent, int)} that {@code
* serviceComponent} actually exists and provides
* {@link RecognitionService#SERVICE_INTERFACE}. Normally you would not use this; call
* {@link #createSpeechRecognizer(Context)} to use the system default recognition
* service instead or {@link #createOnDeviceSpeechRecognizer(Context)} to use on-device
* recognition. For apps targeting Android 11 (API level 30) interaction with a speech recognition
* service requires Please note that {@link #setRecognitionListener(RecognitionListener)} should be called
* before dispatching any command to the created {@code SpeechRecognizer}, otherwise no
* notifications will be received.
*
* @param context in which to create {@code SpeechRecognizer}
* @return a new on-device {@code SpeechRecognizer}.
* @throws UnsupportedOperationException iff {@link #isOnDeviceRecognitionAvailable(Context)}
* is false
*/
@NonNull
@MainThread
public static SpeechRecognizer createOnDeviceSpeechRecognizer(@NonNull final Context context) {
if (!isOnDeviceRecognitionAvailable(context)) {
throw new UnsupportedOperationException("On-device recognition is not available");
}
return lenientlyCreateOnDeviceSpeechRecognizer(context);
}
/**
* Helper method to create on-device SpeechRecognizer in tests even when the device does not
* support on-device speech recognition.
*
* @hide
*/
@TestApi
@NonNull
@MainThread
public static SpeechRecognizer createOnDeviceTestingSpeechRecognizer(
@NonNull final Context context) {
return lenientlyCreateOnDeviceSpeechRecognizer(context);
}
@NonNull
@MainThread
private static SpeechRecognizer lenientlyCreateOnDeviceSpeechRecognizer(
@NonNull final Context context) {
if (context == null) {
throw new IllegalArgumentException("Context cannot be null");
}
checkIsCalledFromMainThread();
return new SpeechRecognizer(context, /* onDevice */ true);
}
/**
* Sets the listener that will receive all the callbacks. The previous unfinished commands will
* be executed with the old listener, while any following command will be executed with the new
* listener.
*
* @param listener listener that will receive all the callbacks from the created
* {@link SpeechRecognizer}, this must not be null.
*/
@MainThread
public void setRecognitionListener(RecognitionListener listener) {
checkIsCalledFromMainThread();
putMessage(Message.obtain(mHandler, MSG_CHANGE_LISTENER, listener));
}
/**
* Starts listening for speech. Please note that
* {@link #setRecognitionListener(RecognitionListener)} should be called beforehand, otherwise
* no notifications will be received.
*
* @param recognizerIntent contains parameters for the recognition to be performed. The intent
* may also contain optional extras, see {@link RecognizerIntent}. If these values are
* not set explicitly, default values will be used by the recognizer.
*/
@MainThread
public void startListening(final Intent recognizerIntent) {
if (recognizerIntent == null) {
throw new IllegalArgumentException("intent must not be null");
}
checkIsCalledFromMainThread();
if (DBG) {
Slog.i(TAG, "#startListening called");
if (mService == null) {
Slog.i(TAG, "Connection is not established yet");
}
}
if (mService == null) {
// First time connection: first establish a connection, then dispatch #startListening.
connectToSystemService();
}
putMessage(Message.obtain(mHandler, MSG_START, recognizerIntent));
}
/**
* Stops listening for speech. Speech captured so far will be recognized as if the user had
* stopped speaking at this point.
*
* Note that in the default case, this does not need to be called, as the speech endpointer
* will automatically stop the recognizer listening when it determines speech has completed.
* However, you can manipulate endpointer parameters directly using the intent extras defined in
* {@link RecognizerIntent}, in which case you may sometimes want to manually call this method
* to stop listening sooner.
*
* Upon invocation clients must wait until {@link RecognitionListener#onResults} or
* {@link RecognitionListener#onError} are invoked before calling
* {@link SpeechRecognizer#startListening} again. Otherwise such an attempt would be rejected by
* recognition service.
*
* Please note that
* {@link #setRecognitionListener(RecognitionListener)} should be called beforehand, otherwise
* no notifications will be received.
*/
@MainThread
public void stopListening() {
checkIsCalledFromMainThread();
if (DBG) {
Slog.i(TAG, "#stopListening called");
if (mService == null) {
Slog.i(TAG, "Connection is not established yet");
}
}
putMessage(Message.obtain(mHandler, MSG_STOP));
}
/**
* Cancels the speech recognition. Please note that
* {@link #setRecognitionListener(RecognitionListener)} should be called beforehand, otherwise
* no notifications will be received.
*/
@MainThread
public void cancel() {
checkIsCalledFromMainThread();
putMessage(Message.obtain(mHandler, MSG_CANCEL));
}
/**
* Sets a temporary component to power on-device speech recognizer.
*
* This is only expected to be called in tests, system would reject calls from client apps.
*
* @param componentName name of the component to set temporary replace speech recognizer. {@code
* null} value resets the recognizer to default.
*
* @hide
*/
@TestApi
@RequiresPermission(Manifest.permission.MANAGE_SPEECH_RECOGNITION)
public void setTemporaryOnDeviceRecognizer(@Nullable ComponentName componentName) {
mHandler.sendMessage(
Message.obtain(mHandler, MSG_SET_TEMPORARY_ON_DEVICE_COMPONENT, componentName));
}
private static void checkIsCalledFromMainThread() {
if (Looper.myLooper() != Looper.getMainLooper()) {
throw new RuntimeException(
"SpeechRecognizer should be used only from the application's main thread");
}
}
private void putMessage(Message msg) {
if (mService == null) {
mPendingTasks.offer(msg);
} else {
mHandler.sendMessage(msg);
}
}
/** sends the actual message to the service */
private void handleStartListening(Intent recognizerIntent) {
if (!checkOpenConnection()) {
return;
}
try {
mService.startListening(recognizerIntent, mListener, mContext.getAttributionSource());
if (DBG) Log.d(TAG, "service start listening command succeded");
} catch (final RemoteException e) {
Log.e(TAG, "startListening() failed", e);
mListener.onError(ERROR_CLIENT);
}
}
/** sends the actual message to the service */
private void handleStopMessage() {
if (!checkOpenConnection()) {
return;
}
try {
mService.stopListening(mListener);
if (DBG) Log.d(TAG, "service stop listening command succeded");
} catch (final RemoteException e) {
Log.e(TAG, "stopListening() failed", e);
mListener.onError(ERROR_CLIENT);
}
}
/** sends the actual message to the service */
private void handleCancelMessage() {
if (!checkOpenConnection()) {
return;
}
try {
mService.cancel(mListener, /*isShutdown*/ false);
if (DBG) Log.d(TAG, "service cancel command succeded");
} catch (final RemoteException e) {
Log.e(TAG, "cancel() failed", e);
mListener.onError(ERROR_CLIENT);
}
}
private void handleSetTemporaryComponent(ComponentName componentName) {
if (DBG) {
Log.d(TAG, "handleSetTemporaryComponent, componentName=" + componentName);
}
if (!maybeInitializeManagerService()) {
return;
}
try {
mManagerService.setTemporaryComponent(componentName);
} catch (final RemoteException e) {
e.rethrowFromSystemServer();
}
}
private boolean checkOpenConnection() {
if (mService != null) {
return true;
}
mListener.onError(ERROR_CLIENT);
Log.e(TAG, "not connected to the recognition service");
return false;
}
/** changes the listener */
private void handleChangeListener(RecognitionListener listener) {
if (DBG) Log.d(TAG, "handleChangeListener, listener=" + listener);
mListener.mInternalListener = listener;
}
/** Destroys the {@code SpeechRecognizer} object. */
public void destroy() {
if (mService != null) {
try {
mService.cancel(mListener, /*isShutdown*/ true);
} catch (final RemoteException e) {
// Not important
}
}
mService = null;
mPendingTasks.clear();
mListener.mInternalListener = null;
}
/** Establishes a connection to system server proxy and initializes the session. */
private void connectToSystemService() {
if (!maybeInitializeManagerService()) {
return;
}
ComponentName componentName = getSpeechRecognizerComponentName();
if (!mOnDevice && componentName == null) {
mListener.onError(ERROR_CLIENT);
return;
}
try {
mManagerService.createSession(
componentName,
mClientToken,
mOnDevice,
new IRecognitionServiceManagerCallback.Stub(){
@Override
public void onSuccess(IRecognitionService service) throws RemoteException {
if (DBG) {
Log.i(TAG, "Connected to speech recognition service");
}
mService = service;
while (!mPendingTasks.isEmpty()) {
mHandler.sendMessage(mPendingTasks.poll());
}
}
@Override
public void onError(int errorCode) throws RemoteException {
Log.e(TAG, "Bind to system recognition service failed with error "
+ errorCode);
mListener.onError(errorCode);
}
});
} catch (RemoteException e) {
e.rethrowFromSystemServer();
}
}
private boolean maybeInitializeManagerService() {
if (mManagerService != null) {
return true;
}
mManagerService = IRecognitionServiceManager.Stub.asInterface(
ServiceManager.getService(Context.SPEECH_RECOGNITION_SERVICE));
if (mManagerService == null && mListener != null) {
mListener.onError(ERROR_CLIENT);
return false;
}
return true;
}
/**
* Returns the component name to be used for establishing a connection, based on the parameters
* used during initialization.
*
* Note the 3 different scenarios:
* {@code
*
*
* @param context in which to create {@code SpeechRecognizer}
* @return a new {@code SpeechRecognizer}
*/
@MainThread
public static SpeechRecognizer createSpeechRecognizer(final Context context) {
return createSpeechRecognizer(context, null);
}
/**
* Factory method to create a new {@code SpeechRecognizer}. Please note that
* {@link #setRecognitionListener(RecognitionListener)} should be called before dispatching any
* command to the created {@code SpeechRecognizer}, otherwise no notifications will be
* received.
* Use this version of the method to specify a specific service to direct this
* {@link SpeechRecognizer} to.
*
* {@code
*
*
* @param context in which to create {@code SpeechRecognizer}
* @param serviceComponent the {@link ComponentName} of a specific service to direct this
* {@code SpeechRecognizer} to
* @return a new {@code SpeechRecognizer}
*/
@MainThread
public static SpeechRecognizer createSpeechRecognizer(final Context context,
final ComponentName serviceComponent) {
if (context == null) {
throw new IllegalArgumentException("Context cannot be null");
}
checkIsCalledFromMainThread();
return new SpeechRecognizer(context, serviceComponent);
}
/**
* Factory method to create a new {@code SpeechRecognizer}.
*
*
*