xref: /packages/inputmethods/LatinIME/java/src/com/android/inputmethod/voice/VoiceInput.java

/*
 * Copyright (C) 2009 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package com.android.inputmethod.voice;

import com.android.inputmethod.latin.EditingUtil;
import com.android.inputmethod.latin.R;

import android.content.ContentResolver;
import android.content.Context;
import android.content.Intent;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Message;
import android.os.Parcelable;
import android.speech.RecognitionListener;
import android.speech.SpeechRecognizer;
import android.speech.RecognizerIntent;
import android.util.Log;
import android.view.inputmethod.InputConnection;
import android.view.View;
import android.view.View.OnClickListener;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

/**
 * Speech recognition input, including both user interface and a background
 * process to stream audio to the network recognizer. This class supplies a
 * View (getView()), which it updates as recognition occurs. The user of this
 * class is responsible for making the view visible to the user, as well as
 * handling various events returned through UiListener.
 */
public class VoiceInput implements OnClickListener {
    private static final String TAG = "VoiceInput";
    private static final String EXTRA_RECOGNITION_CONTEXT =
            "android.speech.extras.RECOGNITION_CONTEXT";
    private static final String EXTRA_CALLING_PACKAGE = "calling_package";
    private static final String EXTRA_ALTERNATES = "android.speech.extra.ALTERNATES";
    private static final int MAX_ALT_LIST_LENGTH = 6;

    private static final String DEFAULT_RECOMMENDED_PACKAGES =
            "com.android.mms " +
            "com.google.android.gm " +
            "com.google.android.talk " +
            "com.google.android.apps.googlevoice " +
            "com.android.email " +
            "com.android.browser ";

    // WARNING! Before enabling this, fix the problem with calling getExtractedText() in
    // landscape view. It causes Extracted text updates to be rejected due to a token mismatch
    public static boolean ENABLE_WORD_CORRECTIONS = true;

    // Dummy word suggestion which means "delete current word"
    public static final String DELETE_SYMBOL = " \u00D7 ";  // times symbol

    private Whitelist mRecommendedList;
    private Whitelist mBlacklist;

    private VoiceInputLogger mLogger;

    // Names of a few extras defined in VoiceSearch's RecognitionController
    // Note, the version of voicesearch that shipped in Froyo returns the raw
    // RecognitionClientAlternates protocol buffer under the key "alternates",
    // so a VS market update must be installed on Froyo devices in order to see
    // alternatives.
    private static final String ALTERNATES_BUNDLE = "alternates_bundle";

    //  This is copied from the VoiceSearch app.
    private static final class AlternatesBundleKeys {
        public static final String ALTERNATES = "alternates";
        public static final String CONFIDENCE = "confidence";
        public static final String LENGTH = "length";
        public static final String MAX_SPAN_LENGTH = "max_span_length";
        public static final String SPANS = "spans";
        public static final String SPAN_KEY_DELIMITER = ":";
        public static final String START = "start";
        public static final String TEXT = "text";
    }

    // Names of a few intent extras defined in VoiceSearch's RecognitionService.
    // These let us tweak the endpointer parameters.
    private static final String EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS =
            "android.speech.extras.SPEECH_INPUT_MINIMUM_LENGTH_MILLIS";
    private static final String EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS =
            "android.speech.extras.SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS";
    private static final String EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS =
            "android.speech.extras.SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS";

    // The usual endpointer default value for input complete silence length is 0.5 seconds,
    // but that's used for things like voice search. For dictation-like voice input like this,
    // we go with a more liberal value of 1 second. This value will only be used if a value
    // is not provided from Gservices.
    private static final String INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS = "1000";

    // Used to record part of that state for logging purposes.
    public static final int DEFAULT = 0;
    public static final int LISTENING = 1;
    public static final int WORKING = 2;
    public static final int ERROR = 3;

    private int mAfterVoiceInputDeleteCount = 0;
    private int mAfterVoiceInputInsertCount = 0;
    private int mAfterVoiceInputInsertPunctuationCount = 0;
    private int mAfterVoiceInputCursorPos = 0;
    private int mAfterVoiceInputSelectionSpan = 0;

    private int mState = DEFAULT;

    private final static int MSG_CLOSE_ERROR_DIALOG = 1;

    private final Handler mHandler = new Handler() {
        @Override
        public void handleMessage(Message msg) {
            if (msg.what == MSG_CLOSE_ERROR_DIALOG) {
                mState = DEFAULT;
                mRecognitionView.finish();
                mUiListener.onCancelVoice();
            }
        }
    };

    /**
     * Events relating to the recognition UI. You must implement these.
     */
    public interface UiListener {

        /**
         * @param recognitionResults a set of transcripts for what the user
         *   spoke, sorted by likelihood.
         */
        public void onVoiceResults(
            List<String> recognitionResults,
            Map<String, List<CharSequence>> alternatives);

        /**
         * Called when the user cancels speech recognition.
         */
        public void onCancelVoice();
    }

    private SpeechRecognizer mSpeechRecognizer;
    private RecognitionListener mRecognitionListener;
    private RecognitionView mRecognitionView;
    private UiListener mUiListener;
    private Context mContext;

    /**
     * @param context the service or activity in which we're running.
     * @param uiHandler object to receive events from VoiceInput.
     */
    public VoiceInput(Context context, UiListener uiHandler) {
        mLogger = VoiceInputLogger.getLogger(context);
        mRecognitionListener = new ImeRecognitionListener();
        mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(context);
        mSpeechRecognizer.setRecognitionListener(mRecognitionListener);
        mUiListener = uiHandler;
        mContext = context;
        newView();

        String recommendedPackages = SettingsUtil.getSettingsString(
                context.getContentResolver(),
                SettingsUtil.LATIN_IME_VOICE_INPUT_RECOMMENDED_PACKAGES,
                DEFAULT_RECOMMENDED_PACKAGES);

        mRecommendedList = new Whitelist();
        for (String recommendedPackage : recommendedPackages.split("\\s+")) {
            mRecommendedList.addApp(recommendedPackage);
        }

        mBlacklist = new Whitelist();
        mBlacklist.addApp("com.android.setupwizard");
    }

    public void setCursorPos(int pos) {
        mAfterVoiceInputCursorPos = pos;
    }

    public int getCursorPos() {
        return mAfterVoiceInputCursorPos;
    }

    public void setSelectionSpan(int span) {
        mAfterVoiceInputSelectionSpan = span;
    }

    public int getSelectionSpan() {
        return mAfterVoiceInputSelectionSpan;
    }

    public void incrementTextModificationDeleteCount(int count){
        mAfterVoiceInputDeleteCount += count;
        // Send up intents for other text modification types
        if (mAfterVoiceInputInsertCount > 0) {
            logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount);
            mAfterVoiceInputInsertCount = 0;
        }
        if (mAfterVoiceInputInsertPunctuationCount > 0) {
            logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount);
            mAfterVoiceInputInsertPunctuationCount = 0;
        }

    }

    public void incrementTextModificationInsertCount(int count){
        mAfterVoiceInputInsertCount += count;
        if (mAfterVoiceInputSelectionSpan > 0) {
            // If text was highlighted before inserting the char, count this as
            // a delete.
            mAfterVoiceInputDeleteCount += mAfterVoiceInputSelectionSpan;
        }
        // Send up intents for other text modification types
        if (mAfterVoiceInputDeleteCount > 0) {
            logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount);
            mAfterVoiceInputDeleteCount = 0;
        }
        if (mAfterVoiceInputInsertPunctuationCount > 0) {
            logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount);
            mAfterVoiceInputInsertPunctuationCount = 0;
        }
    }

    public void incrementTextModificationInsertPunctuationCount(int count){
        mAfterVoiceInputInsertPunctuationCount += 1;
        if (mAfterVoiceInputSelectionSpan > 0) {
            // If text was highlighted before inserting the char, count this as
            // a delete.
            mAfterVoiceInputDeleteCount += mAfterVoiceInputSelectionSpan;
        }
        // Send up intents for aggregated non-punctuation insertions
        if (mAfterVoiceInputDeleteCount > 0) {
            logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount);
            mAfterVoiceInputDeleteCount = 0;
        }
        if (mAfterVoiceInputInsertCount > 0) {
            logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount);
            mAfterVoiceInputInsertCount = 0;
        }
    }

    public void flushAllTextModificationCounters() {
        if (mAfterVoiceInputInsertCount > 0) {
            logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount);
            mAfterVoiceInputInsertCount = 0;
        }
        if (mAfterVoiceInputDeleteCount > 0) {
            logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount);
            mAfterVoiceInputDeleteCount = 0;
        }
        if (mAfterVoiceInputInsertPunctuationCount > 0) {
            logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount);
            mAfterVoiceInputInsertPunctuationCount = 0;
        }
    }

    /**
     * The configuration of the IME changed and may have caused the views to be layed out
     * again. Restore the state of the recognition view.
     */
    public void onConfigurationChanged() {
        mRecognitionView.restoreState();
    }

    /**
     * @return true if field is blacklisted for voice
     */
    public boolean isBlacklistedField(FieldContext context) {
        return mBlacklist.matches(context);
    }

    /**
     * Used to decide whether to show voice input hints for this field, etc.
     *
     * @return true if field is recommended for voice
     */
    public boolean isRecommendedField(FieldContext context) {
        return mRecommendedList.matches(context);
    }

    /**
     * Start listening for speech from the user. This will grab the microphone
     * and start updating the view provided by getView(). It is the caller's
     * responsibility to ensure that the view is visible to the user at this stage.
     *
     * @param context the same FieldContext supplied to voiceIsEnabled()
     * @param swipe whether this voice input was started by swipe, for logging purposes
     */
    public void startListening(FieldContext context, boolean swipe) {
        mState = DEFAULT;

        Locale locale = Locale.getDefault();
        String localeString = locale.getLanguage() + "-" + locale.getCountry();

        mLogger.start(localeString, swipe);

        mState = LISTENING;

        mRecognitionView.showInitializing();
        startListeningAfterInitialization(context);
    }

    /**
     * Called only when the recognition manager's initialization completed
     *
     * @param context context with which {@link #startListening(FieldContext, boolean)} was executed
     */
    private void startListeningAfterInitialization(FieldContext context) {
        Intent intent = makeIntent();
        intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, "");
        intent.putExtra(EXTRA_RECOGNITION_CONTEXT, context.getBundle());
        intent.putExtra(EXTRA_CALLING_PACKAGE, "VoiceIME");
        intent.putExtra(EXTRA_ALTERNATES, true);
        intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS,
                SettingsUtil.getSettingsInt(
                        mContext.getContentResolver(),
                        SettingsUtil.LATIN_IME_MAX_VOICE_RESULTS,
                        1));
        // Get endpointer params from Gservices.
        // TODO: Consider caching these values for improved performance on slower devices.
        final ContentResolver cr = mContext.getContentResolver();
        putEndpointerExtra(
                cr,
                intent,
                SettingsUtil.LATIN_IME_SPEECH_MINIMUM_LENGTH_MILLIS,
                EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS,
                null  /* rely on endpointer default */);
        putEndpointerExtra(
                cr,
                intent,
                SettingsUtil.LATIN_IME_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
                EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
                INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS
                /* our default value is different from the endpointer's */);
        putEndpointerExtra(
                cr,
                intent,
                SettingsUtil.
                        LATIN_IME_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
                EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
                null  /* rely on endpointer default */);

        mSpeechRecognizer.startListening(intent);
    }

    /**
     * Gets the value of the provided Gservices key, attempts to parse it into a long,
     * and if successful, puts the long value as an extra in the provided intent.
     */
    private void putEndpointerExtra(ContentResolver cr, Intent i,
            String gservicesKey, String intentExtraKey, String defaultValue) {
        long l = -1;
        String s = SettingsUtil.getSettingsString(cr, gservicesKey, defaultValue);
        if (s != null) {
            try {
                l = Long.valueOf(s);
            } catch (NumberFormatException e) {
                Log.e(TAG, "could not parse value for " + gservicesKey + ": " + s);
            }
        }

        if (l != -1) i.putExtra(intentExtraKey, l);
    }

    public void destroy() {
        mSpeechRecognizer.destroy();
    }

    /**
     * Creates a new instance of the view that is returned by {@link #getView()}
     * Clients should use this when a previously returned view is stuck in a
     * layout that is being thrown away and a new one is need to show to the
     * user.
     */
    public void newView() {
        mRecognitionView = new RecognitionView(mContext, this);
    }

    /**
     * @return a view that shows the recognition flow--e.g., "Speak now" and
     * "working" dialogs.
     */
    public View getView() {
        return mRecognitionView.getView();
    }

    /**
     * Handle the cancel button.
     */
    public void onClick(View view) {
        switch(view.getId()) {
            case R.id.button:
                cancel();
                break;
        }
    }

    public void logTextModifiedByTypingInsertion(int length) {
        mLogger.textModifiedByTypingInsertion(length);
    }

    public void logTextModifiedByTypingInsertionPunctuation(int length) {
        mLogger.textModifiedByTypingInsertionPunctuation(length);
    }

    public void logTextModifiedByTypingDeletion(int length) {
        mLogger.textModifiedByTypingDeletion(length);
    }

    public void logTextModifiedByChooseSuggestion(String suggestion, int index,
                                                  String wordSeparators, InputConnection ic) {
        EditingUtil.Range range = new EditingUtil.Range();
        String wordToBeReplaced = EditingUtil.getWordAtCursor(ic, wordSeparators, range);
        // If we enable phrase-based alternatives, only send up the first word
        // in suggestion and wordToBeReplaced.
        mLogger.textModifiedByChooseSuggestion(suggestion.length(), wordToBeReplaced.length(),
                                               index, wordToBeReplaced, suggestion);
    }

    public void logKeyboardWarningDialogShown() {
        mLogger.keyboardWarningDialogShown();
    }

    public void logKeyboardWarningDialogDismissed() {
        mLogger.keyboardWarningDialogDismissed();
    }

    public void logKeyboardWarningDialogOk() {
        mLogger.keyboardWarningDialogOk();
    }

    public void logKeyboardWarningDialogCancel() {
        mLogger.keyboardWarningDialogCancel();
    }

    public void logSwipeHintDisplayed() {
        mLogger.swipeHintDisplayed();
    }

    public void logPunctuationHintDisplayed() {
        mLogger.punctuationHintDisplayed();
    }

    public void logVoiceInputDelivered(int length) {
        mLogger.voiceInputDelivered(length);
    }

    public void logInputEnded() {
        mLogger.inputEnded();
    }

    public void flushLogs() {
        mLogger.flush();
    }

    private static Intent makeIntent() {
        Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);

        // On Cupcake, use VoiceIMEHelper since VoiceSearch doesn't support.
        // On Donut, always use VoiceSearch, since VoiceIMEHelper and
        // VoiceSearch may conflict.
        if (Build.VERSION.RELEASE.equals("1.5")) {
            intent = intent.setClassName(
              "com.google.android.voiceservice",
              "com.google.android.voiceservice.IMERecognitionService");
        } else {
            intent = intent.setClassName(
              "com.google.android.voicesearch",
              "com.google.android.voicesearch.RecognitionService");
        }

        return intent;
    }

    /**
     * Cancel in-progress speech recognition.
     */
    public void cancel() {
        switch (mState) {
        case LISTENING:
            mLogger.cancelDuringListening();
            break;
        case WORKING:
            mLogger.cancelDuringWorking();
            break;
        case ERROR:
            mLogger.cancelDuringError();
            break;
        }
        mState = DEFAULT;

        // Remove all pending tasks (e.g., timers to cancel voice input)
        mHandler.removeMessages(MSG_CLOSE_ERROR_DIALOG);

        mSpeechRecognizer.cancel();
        mUiListener.onCancelVoice();
        mRecognitionView.finish();
    }

    private int getErrorStringId(int errorType, boolean endpointed) {
        switch (errorType) {
            // We use CLIENT_ERROR to signify that voice search is not available on the device.
            case SpeechRecognizer.ERROR_CLIENT:
                return R.string.voice_not_installed;
            case SpeechRecognizer.ERROR_NETWORK:
                return R.string.voice_network_error;
            case SpeechRecognizer.ERROR_NETWORK_TIMEOUT:
                return endpointed ?
                        R.string.voice_network_error : R.string.voice_too_much_speech;
            case SpeechRecognizer.ERROR_AUDIO:
                return R.string.voice_audio_error;
            case SpeechRecognizer.ERROR_SERVER:
                return R.string.voice_server_error;
            case SpeechRecognizer.ERROR_SPEECH_TIMEOUT:
                return R.string.voice_speech_timeout;
            case SpeechRecognizer.ERROR_NO_MATCH:
                return R.string.voice_no_match;
            default: return R.string.voice_error;
        }
    }

    private void onError(int errorType, boolean endpointed) {
        Log.i(TAG, "error " + errorType);
        mLogger.error(errorType);
        onError(mContext.getString(getErrorStringId(errorType, endpointed)));
    }

    private void onError(String error) {
        mState = ERROR;
        mRecognitionView.showError(error);
        // Wait a couple seconds and then automatically dismiss message.
        mHandler.sendMessageDelayed(Message.obtain(mHandler, MSG_CLOSE_ERROR_DIALOG), 2000);
    }

    private class ImeRecognitionListener implements RecognitionListener {
        // Waveform data
        final ByteArrayOutputStream mWaveBuffer = new ByteArrayOutputStream();
        int mSpeechStart;
        private boolean mEndpointed = false;

        public void onReadyForSpeech(Bundle noiseParams) {
            mRecognitionView.showListening();
        }

        public void onBeginningOfSpeech() {
            mEndpointed = false;
            mSpeechStart = mWaveBuffer.size();
        }

        public void onRmsChanged(float rmsdB) {
            mRecognitionView.updateVoiceMeter(rmsdB);
        }

        public void onBufferReceived(byte[] buf) {
            try {
                mWaveBuffer.write(buf);
            } catch (IOException e) {}
        }

        public void onEndOfSpeech() {
            mEndpointed = true;
            mState = WORKING;
            mRecognitionView.showWorking(mWaveBuffer, mSpeechStart, mWaveBuffer.size());
        }

        public void onError(int errorType) {
            mState = ERROR;
            VoiceInput.this.onError(errorType, mEndpointed);
        }

        public void onResults(Bundle resultsBundle) {
            List<String> results = resultsBundle
                    .getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
            // VS Market update is needed for IME froyo clients to access the alternatesBundle
            // TODO: verify this.
            Bundle alternatesBundle = resultsBundle.getBundle(ALTERNATES_BUNDLE);
            mState = DEFAULT;

            final Map<String, List<CharSequence>> alternatives =
                new HashMap<String, List<CharSequence>>();

            if (ENABLE_WORD_CORRECTIONS && alternatesBundle != null && results.size() > 0) {
                // Use the top recognition result to map each alternative's start:length to a word.
                String[] words = results.get(0).split(" ");
                Bundle spansBundle = alternatesBundle.getBundle(AlternatesBundleKeys.SPANS);
                for (String key : spansBundle.keySet()) {
                    // Get the word for which these alternates correspond to.
                    Bundle spanBundle = spansBundle.getBundle(key);
                    int start = spanBundle.getInt(AlternatesBundleKeys.START);
                    int length = spanBundle.getInt(AlternatesBundleKeys.LENGTH);
                    // Only keep single-word based alternatives.
                    if (length == 1 && start < words.length) {
                        // Get the alternatives associated with the span.
                        // If a word appears twice in a recognition result,
                        // concatenate the alternatives for the word.
                        List<CharSequence> altList = alternatives.get(words[start]);
                        if (altList == null) {
                            altList = new ArrayList<CharSequence>();
                            alternatives.put(words[start], altList);
                        }
                        Parcelable[] alternatesArr = spanBundle
                            .getParcelableArray(AlternatesBundleKeys.ALTERNATES);
                        for (int j = 0; j < alternatesArr.length &&
                                 altList.size() < MAX_ALT_LIST_LENGTH; j++) {
                            Bundle alternateBundle = (Bundle) alternatesArr[j];
                            String alternate = alternateBundle.getString(AlternatesBundleKeys.TEXT);
                            // Don't allow duplicates in the alternates list.
                            if (!altList.contains(alternate)) {
                                altList.add(alternate);
                            }
                        }
                    }
                }
            }

            if (results.size() > 5) {
                results = results.subList(0, 5);
            }
            mUiListener.onVoiceResults(results, alternatives);
            mRecognitionView.finish();
        }

        public void onPartialResults(final Bundle partialResults) {
            // currently - do nothing
        }

        public void onEvent(int eventType, Bundle params) {
            // do nothing - reserved for events that might be added in the future
        }
    }
}