1 /** 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.car.voicecontrol; 17 18 import static com.android.car.voicecontrol.DirectSendUtils.AMBIGUOUS_RESULT; 19 20 import android.annotation.StringRes; 21 import android.app.Activity; 22 import android.app.AlertDialog; 23 import android.app.Notification; 24 import android.app.PendingIntent; 25 import android.bluetooth.BluetoothDevice; 26 import android.content.Context; 27 import android.content.Intent; 28 import android.content.pm.ApplicationInfo; 29 import android.content.pm.PackageManager; 30 import android.os.Bundle; 31 import android.os.Handler; 32 import android.util.Log; 33 import android.widget.EditText; 34 import android.widget.LinearLayout; 35 36 import androidx.annotation.Nullable; 37 import androidx.core.app.NotificationCompat; 38 39 import com.android.car.TelecomUtils; 40 import com.android.car.assist.CarVoiceInteractionSession; 41 import com.android.car.assist.payloadhandlers.ConversationPayloadHandler; 42 import com.android.car.assist.payloadhandlers.NotificationPayloadHandler; 43 import com.android.car.messenger.common.Conversation; 44 import com.android.car.telephony.common.Contact; 45 import com.android.car.telephony.common.PhoneNumber; 46 import com.android.car.voicecontrol.actuators.SkillRouter; 47 48 import java.util.ArrayList; 49 import java.util.List; 50 51 /** 52 * Launcher activity for this sample voice interaction service. This allows voice interaction to be 53 * launched from the app launcher. 54 */ 55 public class VoicePlateActivity extends Activity 56 implements SpeechToText.Listener, TextToSpeech.Listener { 57 public static final String EXTRA_ACTION = "com.android.car.voicecontrol.ACTION"; 58 public static final String EXTRA_ARGS = "com.android.car.voicecontrol.ARGS"; 59 private static final String TAG = "Mica.MainActivity"; 60 private SpeechToText mSTT; 61 private TextToSpeech mTTS; 62 private VoicePlateController mPresenter; 63 private SkillRouter mActuator; 64 private Handler mHandler = new Handler(); 65 private NotificationPayloadHandler mNotifHandler; 66 private InteractionServiceClient mInteractionService; 67 private TelecomUtils mTelecomUtils; 68 69 @Override onCreate(@ullable Bundle savedInstanceState)70 protected void onCreate(@Nullable Bundle savedInstanceState) { 71 super.onCreate(savedInstanceState); 72 mPresenter = new VoicePlateController(this, getLayoutInflater()); 73 setContentView(mPresenter.getContentView()); 74 mSTT = new SpeechToTextImpl(this); 75 mTTS = new TextToSpeechImpl(this, this); 76 mActuator = new SkillRouter(this); 77 mNotifHandler = new NotificationPayloadHandler(this); 78 mInteractionService = new InteractionServiceClient(this) { 79 @Override 80 void onConnected() { 81 mTTS.setSelectedVoice(mInteractionService.getVoice()); 82 } 83 }; 84 mInteractionService.connect(); 85 86 mTelecomUtils = new TelecomUtils(getApplicationContext()); 87 } 88 89 @Override onDestroy()90 protected void onDestroy() { 91 mInteractionService.disconnect(); 92 mSTT.destroy(); 93 mTTS.destroy(); 94 mActuator.destroy(); 95 super.onDestroy(); 96 } 97 98 @Override onStart()99 protected void onStart() { 100 super.onStart(); 101 mPresenter.updateRecognizedText(""); 102 mPresenter.updateState(VoicePlateController.State.LISTENING); 103 104 if (!InteractionService.hasAllPermissions(this)) { 105 mTTS.speak(R.string.speech_reply_missing_permissions); 106 mPresenter.updateState(VoicePlateController.State.DELIVERING); 107 mInteractionService.notifySetupChanged(); 108 return; 109 } 110 111 Intent intent = getIntent(); 112 String action = intent.getStringExtra(EXTRA_ACTION); 113 if (CarVoiceInteractionSession.VOICE_ACTION_READ_NOTIFICATION.equals(action)) { 114 Notification notification = 115 mNotifHandler.getNotification(intent.getBundleExtra(EXTRA_ARGS)); 116 onReadNotification(notification); 117 return; 118 } 119 if (CarVoiceInteractionSession.VOICE_ACTION_REPLY_NOTIFICATION.equals(action)) { 120 Notification notification = 121 mNotifHandler.getNotification(intent.getBundleExtra(EXTRA_ARGS)); 122 onReplyNotification(notification); 123 return; 124 } 125 if (CarVoiceInteractionSession.VOICE_ACTION_HANDLE_EXCEPTION.equals(action)) { 126 onReadNotificationException(intent.getBundleExtra(EXTRA_ARGS)); 127 return; 128 } 129 if (CarVoiceInteractionSession.VOICE_ACTION_READ_CONVERSATION.equals(action)) { 130 onReadConversation(intent.getBundleExtra(EXTRA_ARGS)); 131 return; 132 } 133 if (CarVoiceInteractionSession.VOICE_ACTION_REPLY_CONVERSATION.equals(action)) { 134 onReplyConversation(intent.getBundleExtra(EXTRA_ARGS)); 135 return; 136 } 137 if (CarVoiceInteractionSession.VOICE_ACTION_SEND_SMS.equals(action)) { 138 onDirectSendSMS(intent.getBundleExtra(EXTRA_ARGS)); 139 return; 140 } 141 mSTT.startListening(this); 142 } 143 ask(TextToSpeech.QuestionCallback callback, @StringRes int resId, Object... args)144 private void ask(TextToSpeech.QuestionCallback callback, @StringRes int resId, Object... args) { 145 ask(callback, getString(resId), args); 146 } 147 ask(TextToSpeech.QuestionCallback callback, String str, Object... args)148 private void ask(TextToSpeech.QuestionCallback callback, String str, Object... args) { 149 if (PreferencesController.getInstance(this).isDirectSendTextInput()) { 150 final EditText input = new EditText(this); 151 LinearLayout.LayoutParams lp = new LinearLayout.LayoutParams( 152 LinearLayout.LayoutParams.MATCH_PARENT, 153 LinearLayout.LayoutParams.MATCH_PARENT); 154 input.setLayoutParams(lp); 155 156 String confirm = getString(R.string.debug_dialog_confirm); 157 String cancel = getString(R.string.debug_dialog_cancel); 158 159 AlertDialog alertDialog = new AlertDialog.Builder(this) 160 .setMessage(str) 161 .setPositiveButton(confirm, (dialog, which) -> { 162 List<String> result = new ArrayList<>(); 163 result.add(input.getText().toString()); 164 callback.onResult(result); 165 }) 166 .setNegativeButton(cancel, (dialog, which) -> { 167 List<String> result = new ArrayList<>(); 168 callback.onResult(result); 169 }) 170 .setView(input) 171 .create(); 172 173 alertDialog.show(); 174 } else { 175 mTTS.ask(callback, str, args); 176 } 177 } 178 askToSendSMS(PendingIntent pendingIntent, String deviceAddress, String number, String name)179 private void askToSendSMS(PendingIntent pendingIntent, 180 String deviceAddress, String number, String name) { 181 mPresenter.updateRecognizedText(getString(R.string.speech_reply_request_message)); 182 ask(strings -> { 183 if (strings.isEmpty()) { 184 mTTS.speak(R.string.speech_reply_not_recognized); 185 mPresenter.updateRecognizedText(getString(R.string.speech_reply_not_recognized)); 186 return; 187 } 188 189 String message = strings.get(0); 190 String speech = getString(R.string.speech_reply_okay_sending_sms, name); 191 mTTS.speak(speech); 192 mPresenter.updateRecognizedText(speech); 193 194 DirectSendUtils.sendSMS(getApplicationContext(), 195 pendingIntent, deviceAddress, number, message); 196 }, R.string.speech_reply_request_message); 197 } 198 askToDisambiguatePhoneNumber(DirectSendUtils.ResultsCallback callback, String deviceAddress)199 private void askToDisambiguatePhoneNumber(DirectSendUtils.ResultsCallback callback, 200 String deviceAddress) { 201 mPresenter.updateRecognizedText(getString(R.string.speech_reply_request_contact)); 202 ask(strings -> { 203 if (strings.isEmpty()) { 204 callback.onFailure(getString(R.string.speech_reply_unrecognized_contact)); 205 return; 206 } 207 Contact contact = mInteractionService.getContact(strings.get(0), deviceAddress); 208 if (contact == null) { 209 callback.onFailure(getString(R.string.speech_reply_unrecognized_contact)); 210 return; 211 } 212 213 if (contact.getNumbers().size() == 0) { 214 callback.onFailure(getString(R.string.speech_reply_unrecognized_contact)); 215 } else if (contact.getNumbers().size() == 1) { 216 callback.onSuccess(contact.getNumbers().get(0).getRawNumber(), 217 contact.getDisplayName()); 218 } else if (contact.getNumbers().size() > 1) { 219 String numberAskString = DirectSendUtils.formatNumberAskString( 220 getApplicationContext(), contact.getNumbers()); 221 mPresenter.updateRecognizedText(numberAskString); 222 ask(strings2 -> { 223 if (strings2.isEmpty()) { 224 callback.onFailure(getString(R.string.speech_reply_unrecognized_number)); 225 return; 226 } 227 228 String input = strings2.get(0).toLowerCase(); 229 String[] ordinals = getResources().getStringArray( 230 R.array.speech_reply_ordinals); 231 PhoneNumber phoneNumber; 232 try { 233 if (ordinals[0].equals(input)) { 234 phoneNumber = contact.getNumbers().get(0); 235 } else if (ordinals[1].equals(input)) { 236 phoneNumber = contact.getNumbers().get(1); 237 } else if (ordinals[2].equals(input)) { 238 phoneNumber = contact.getNumbers().get(2); 239 } else { 240 callback.onFailure( 241 getString(R.string.speech_reply_unrecognized_number)); 242 return; 243 } 244 callback.onSuccess(phoneNumber.getRawNumber(), contact.getDisplayName()); 245 } catch (IndexOutOfBoundsException ex) { 246 Log.w(TAG, "Specified phone number index out of list bounds. " + ex); 247 callback.onFailure(getString(R.string.speech_reply_unrecognized_number)); 248 } 249 }, numberAskString); 250 } 251 }, R.string.speech_reply_request_contact); 252 } 253 askToDisambiguateDeviceAddress(DirectSendUtils.ResultsCallback callback)254 private void askToDisambiguateDeviceAddress(DirectSendUtils.ResultsCallback callback) { 255 List<BluetoothDevice> devices = mTelecomUtils.getHfpDeviceList(); 256 String deviceAskString = DirectSendUtils.formatDeviceAskString(getApplicationContext(), 257 devices); 258 mPresenter.updateRecognizedText(deviceAskString); 259 260 ask(strings -> { 261 String disambiguatedAddress = null; 262 263 if (strings.isEmpty()) { 264 callback.onFailure(getString(R.string.speech_reply_unrecognized_device)); 265 return; 266 } 267 268 try { 269 String input = strings.get(0).toLowerCase(); 270 String[] ordinals = getResources().getStringArray(R.array.speech_reply_ordinals); 271 272 if (ordinals[0].equals(input)) { 273 disambiguatedAddress = devices.get(0).getAddress(); 274 } else if (ordinals[1].equals(input)) { 275 disambiguatedAddress = devices.get(1).getAddress(); 276 } else if (ordinals[2].equals(input)) { 277 disambiguatedAddress = devices.get(2).getAddress(); 278 } else { 279 callback.onFailure(getString(R.string.speech_reply_unrecognized_device)); 280 return; 281 } 282 283 callback.onSuccess(disambiguatedAddress); 284 } catch (NullPointerException ex) { 285 callback.onFailure(getString(R.string.speech_reply_unrecognized_device)); 286 Log.e(TAG, "Device not found" + ex); 287 } 288 }, deviceAskString); 289 } 290 getDeviceAddress(Bundle args)291 private String getDeviceAddress(Bundle args) { 292 String address = args.getString(CarVoiceInteractionSession.KEY_DEVICE_ADDRESS); 293 if (address != null) { 294 return address; 295 } 296 297 List<BluetoothDevice> devices = mTelecomUtils.getHfpDeviceList(); 298 if (devices.size() == 0) { 299 mPresenter.updateRecognizedText(getString(R.string.speech_reply_unrecognized_device)); 300 mTTS.speak(R.string.speech_reply_unrecognized_device); 301 return null; 302 } else if (devices.size() == 1) { 303 return devices.get(0).getAddress(); 304 } else { 305 return AMBIGUOUS_RESULT; 306 } 307 } 308 309 // Follows a chain of voice interactions to disambiguate phone number and device address. onDirectSendSMS(Bundle args)310 private void onDirectSendSMS(Bundle args) { 311 Log.d(TAG, "Sending sms"); 312 mPresenter.updateRecognizedText("Sending SMS"); 313 mPresenter.updateState(VoicePlateController.State.DELIVERING); 314 315 // Pending Intent is required to send SMS. 316 PendingIntent pendingIntent = args.getParcelable( 317 CarVoiceInteractionSession.KEY_SEND_PENDING_INTENT); 318 if (pendingIntent == null) { 319 Log.d(TAG, "Pending Intent is null"); 320 return; 321 } 322 323 // Address may or may not be provided. 324 String address = getDeviceAddress(args); 325 if (AMBIGUOUS_RESULT.equals(address)) { 326 // If address is not provided, then contact must be looked up. 327 askToDisambiguateDeviceAddress(new DirectSendUtils.ResultsCallback() { 328 @Override 329 public void onSuccess(String... strings) { 330 String deviceAddress = strings[0]; 331 askToDisambiguatePhoneNumber(new DirectSendUtils.ResultsCallback() { 332 @Override 333 public void onSuccess(String... strings) { 334 String phoneNumber = strings[0]; 335 String name = strings[1]; 336 askToSendSMS(pendingIntent, deviceAddress, phoneNumber, name); 337 } 338 @Override 339 public void onFailure(String error) { 340 mPresenter.updateRecognizedText(error); 341 mTTS.speak(error); 342 } 343 }, deviceAddress); 344 } 345 @Override 346 public void onFailure(String error) { 347 mPresenter.updateRecognizedText(error); 348 mTTS.speak(error); 349 } 350 }); 351 return; 352 } 353 354 // Address is provided, number may or may not be provided. 355 String number = args.getString(CarVoiceInteractionSession.KEY_PHONE_NUMBER); 356 if (number == null) { 357 askToDisambiguatePhoneNumber(new DirectSendUtils.ResultsCallback() { 358 @Override 359 public void onSuccess(String... strings) { 360 String phoneNumber = strings[0]; 361 String name = strings[1]; 362 askToSendSMS(pendingIntent, address, phoneNumber, name); 363 } 364 @Override 365 public void onFailure(String error) { 366 mPresenter.updateRecognizedText(error); 367 mTTS.speak(error); 368 } 369 }, address); 370 return; 371 } 372 373 String name = args.getString(CarVoiceInteractionSession.KEY_RECIPIENT_NAME); 374 if (name == null) { 375 name = ""; 376 } 377 askToSendSMS(pendingIntent, address, number, name); 378 } 379 onReplyConversation(Bundle args)380 private void onReplyConversation(Bundle args) { 381 Conversation conversation = Conversation.fromBundle( 382 args.getBundle(CarVoiceInteractionSession.KEY_CONVERSATION)); 383 Notification notification = ConversationPayloadHandler.createNotificationFromConversation( 384 this, "channel_id", conversation, getIconRes(this), null); 385 onReplyNotification(notification); 386 } 387 onReadConversation(Bundle args)388 private void onReadConversation(Bundle args) { 389 Conversation conversation = Conversation.fromBundle( 390 args.getBundle(CarVoiceInteractionSession.KEY_CONVERSATION)); 391 Notification notification = ConversationPayloadHandler.createNotificationFromConversation( 392 this, "channel_id", conversation, getIconRes(this), null); 393 onReadNotification(notification); 394 } 395 getIconRes(Context context)396 private int getIconRes(Context context) { 397 try { 398 PackageManager packageManager = context.getPackageManager(); 399 ApplicationInfo applicationInfo = 400 packageManager.getApplicationInfo( 401 context.getPackageName(), PackageManager.GET_META_DATA); 402 return applicationInfo.icon; 403 } catch (PackageManager.NameNotFoundException e) { 404 //logger.atWarning().log("Package name not found in system"); 405 return 0; 406 } 407 } 408 onReadNotification(Notification notif)409 private void onReadNotification(Notification notif) { 410 Log.d(TAG, "Reading notification"); 411 mPresenter.updateRecognizedText("Reading a notification"); 412 mPresenter.updateState(VoicePlateController.State.DELIVERING); 413 List<NotificationCompat.MessagingStyle.Message> msgs = mNotifHandler.getMessages(notif); 414 if (msgs == null || msgs.isEmpty()) { 415 Log.d(TAG, "Reading notification: no messages"); 416 mTTS.speak(R.string.speech_reply_no_messages_in_notif); 417 return; 418 } 419 420 // Composing read-out 421 StringBuilder sb = new StringBuilder(); 422 if (msgs.size() == 1) { 423 NotificationCompat.MessagingStyle.Message msg = msgs.get(0); 424 Log.d(TAG, "Reading notification: " + msg.getText()); 425 sb.append(getString(R.string.speech_reply_reading_single_notif, 426 msg.getPerson().getName(), msg.getText())); 427 } else { 428 Log.d(TAG, "Reading notification: # msgs: " + msgs.size()); 429 sb.append(R.string.speech_reply_reading_multiple_notif_header); 430 for (NotificationCompat.MessagingStyle.Message msg : msgs) { 431 sb.append(getString(R.string.speech_reply_reading_multiple_notif_item, 432 msg.getPerson().getName(), msg.getText())); 433 } 434 sb.append(R.string.speech_reply_reading_multiple_notif_footer); 435 } 436 437 // Asking for reply, if available 438 if (mNotifHandler.getAction(notif, Notification.Action.SEMANTIC_ACTION_REPLY) != null) { 439 sb.append(getString(R.string.speech_reply_reply_question)); 440 mTTS.ask(mTTS.createBooleanQuestionCallback(affirmative -> { 441 if (affirmative) { 442 onReplyNotification(notif); 443 } else { 444 mPresenter.updateState(VoicePlateController.State.DELIVERING); 445 mTTS.speak(R.string.speech_reply_cancelled_order); 446 } 447 }, getResources().getInteger(R.integer.boolean_question_max_retries), false), 448 sb.toString()); 449 } else { 450 mTTS.speak(sb.toString()); 451 } 452 } 453 onReplyNotification(Notification notif)454 private void onReplyNotification(Notification notif) { 455 Log.d(TAG, "Reply notification"); 456 Notification.Action action = mNotifHandler.getAction(notif, 457 Notification.Action.SEMANTIC_ACTION_REPLY); 458 if (action == null) { 459 mTTS.speak(R.string.speech_reply_cannot_be_replied); 460 return; 461 } 462 mPresenter.updateRecognizedText("Replying a notification"); 463 mPresenter.updateState(VoicePlateController.State.DELIVERING); 464 mTTS.ask(strings -> { 465 mPresenter.updateState(VoicePlateController.State.DELIVERING); 466 if (strings.isEmpty()) { 467 mTTS.speak(R.string.speech_reply_not_recognized); 468 return; 469 } 470 mTTS.speak(R.string.speech_reply_sending_reply); 471 Intent additionalData = mNotifHandler.writeReply(action, strings.get(0)); 472 mNotifHandler.fireAction(action, additionalData); 473 }, R.string.speech_reply_message_question); 474 } 475 onReadNotificationException(Bundle args)476 private void onReadNotificationException(Bundle args) { 477 if (CarVoiceInteractionSession.EXCEPTION_NOTIFICATION_LISTENER_PERMISSIONS_MISSING 478 .equals(args.getString(CarVoiceInteractionSession.KEY_EXCEPTION))) { 479 mTTS.speak(R.string.speech_reply_missing_notif_access); 480 mInteractionService.notifySetupChanged(); 481 } else { 482 mTTS.speak(R.string.speech_reply_error_reading_notif); 483 } 484 } 485 486 @Override onStop()487 protected void onStop() { 488 super.onStop(); 489 mPresenter.updateRecognizedText(""); 490 mPresenter.updateState(VoicePlateController.State.IDLE); 491 } 492 493 @Override onRecognitionStarted()494 public void onRecognitionStarted() { 495 mPresenter.updateState(VoicePlateController.State.LISTENING_ACTIVE); 496 } 497 498 @Override onPartialRecognition(List<String> strings)499 public void onPartialRecognition(List<String> strings) { 500 mPresenter.updateState(VoicePlateController.State.LISTENING_STREAMING); 501 mPresenter.updateRecognizedText(String.join(", ", strings)); 502 } 503 504 @Override onRecognitionFinished(List<String> strings)505 public void onRecognitionFinished(List<String> strings) { 506 Log.d(TAG, "onRecognitionFinished (results: " + strings + ")"); 507 // If we were waiting for an answer, provide it. 508 if (mTTS.isWaitingForAnswer()) { 509 mTTS.provideAnswer(strings); 510 return; 511 } 512 // Otherwise, execute given command 513 if (!strings.isEmpty()) { 514 mPresenter.updateRecognizedText(String.join(", ", strings)); 515 mPresenter.updateState(VoicePlateController.State.PROCESSING); 516 mHandler.postDelayed(() -> { 517 mPresenter.updateState(VoicePlateController.State.DELIVERING); 518 mActuator.process(strings, mTTS); 519 }, 500); 520 } else { 521 mPresenter.updateState(VoicePlateController.State.DELIVERING); 522 mTTS.speak(R.string.speech_reply_not_recognized); 523 } 524 } 525 526 @Override onUtteranceDone(boolean successful)527 public void onUtteranceDone(boolean successful) { 528 finish(); 529 } 530 531 @Override onWaitingForAnswer()532 public void onWaitingForAnswer() { 533 mPresenter.updateRecognizedText(""); 534 mPresenter.updateState(VoicePlateController.State.LISTENING); 535 mSTT.startListening(this); 536 // The answer will be provided on onRecognitionFinished 537 } 538 } 539