1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.test.voiceinteraction; 18 19 import android.app.ActivityManager; 20 import android.app.VoiceInteractor; 21 import android.app.assist.AssistContent; 22 import android.app.assist.AssistStructure; 23 import android.content.Context; 24 import android.content.Intent; 25 import android.graphics.Bitmap; 26 import android.net.Uri; 27 import android.os.Bundle; 28 import android.service.voice.VoiceInteractionSession; 29 import android.util.Log; 30 import android.view.View; 31 import android.widget.Button; 32 import android.widget.CheckBox; 33 import android.widget.ImageView; 34 import android.widget.TextView; 35 36 public class MainInteractionSession extends VoiceInteractionSession 37 implements View.OnClickListener { 38 static final String TAG = "MainInteractionSession"; 39 40 Intent mStartIntent; 41 View mContentView; 42 AssistVisualizer mAssistVisualizer; 43 View mTopContent; 44 View mBottomContent; 45 TextView mText; 46 Button mTreeButton; 47 Button mTextButton; 48 Button mStartButton; 49 CheckBox mOptionsCheck; 50 View mOptionsContainer; 51 CheckBox mDisallowAssist; 52 CheckBox mDisallowScreenshot; 53 TextView mOptionsText; 54 ImageView mScreenshot; 55 ImageView mFullScreenshot; 56 Button mConfirmButton; 57 Button mCompleteButton; 58 Button mAbortButton; 59 60 AssistStructure mAssistStructure; 61 62 static final int STATE_IDLE = 0; 63 static final int STATE_LAUNCHING = 1; 64 static final int STATE_CONFIRM = 2; 65 static final int STATE_PICK_OPTION = 3; 66 static final int STATE_COMMAND = 4; 67 static final int STATE_ABORT_VOICE = 5; 68 static final int STATE_COMPLETE_VOICE = 6; 69 static final int STATE_DONE = 7; 70 71 int mState = STATE_IDLE; 72 VoiceInteractor.PickOptionRequest.Option[] mPendingOptions; 73 CharSequence mPendingPrompt; 74 Request mPendingRequest; 75 int mCurrentTask = -1; 76 int mShowFlags; 77 MainInteractionSession(Context context)78 MainInteractionSession(Context context) { 79 super(context); 80 } 81 82 @Override onCreate()83 public void onCreate() { 84 super.onCreate(); 85 ActivityManager am = getContext().getSystemService(ActivityManager.class); 86 am.setWatchHeapLimit(40 * 1024 * 1024); 87 } 88 89 @Override onShow(Bundle args, int showFlags)90 public void onShow(Bundle args, int showFlags) { 91 super.onShow(args, showFlags); 92 mShowFlags = showFlags; 93 Log.i(TAG, "onShow: flags=0x" + Integer.toHexString(showFlags) + " args=" + args); 94 mState = STATE_IDLE; 95 mStartIntent = args != null ? (Intent)args.getParcelable("intent") : null; 96 if (mStartIntent == null) { 97 mStartIntent = new Intent(getContext(), TestInteractionActivity.class); 98 } 99 if (mAssistVisualizer != null) { 100 mAssistVisualizer.clearAssistData(); 101 } 102 onHandleScreenshot(null); 103 updateState(); 104 refreshOptions(); 105 } 106 107 @Override onHide()108 public void onHide() { 109 super.onHide(); 110 if (mAssistVisualizer != null) { 111 mAssistVisualizer.clearAssistData(); 112 } 113 mState = STATE_DONE; 114 updateState(); 115 } 116 117 @Override onCreateContentView()118 public View onCreateContentView() { 119 mContentView = getLayoutInflater().inflate(R.layout.voice_interaction_session, null); 120 mAssistVisualizer = (AssistVisualizer)mContentView.findViewById(R.id.assist_visualizer); 121 if (mAssistStructure != null) { 122 mAssistVisualizer.setAssistStructure(mAssistStructure); 123 } 124 mTopContent = mContentView.findViewById(R.id.top_content); 125 mBottomContent = mContentView.findViewById(R.id.bottom_content); 126 mText = (TextView)mContentView.findViewById(R.id.text); 127 mTreeButton = (Button)mContentView.findViewById(R.id.do_tree); 128 mTreeButton.setOnClickListener(this); 129 mTextButton = (Button)mContentView.findViewById(R.id.do_text); 130 mTextButton.setOnClickListener(this); 131 mStartButton = (Button)mContentView.findViewById(R.id.start); 132 mStartButton.setOnClickListener(this); 133 mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot); 134 mScreenshot.setOnClickListener(this); 135 mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot); 136 mOptionsCheck = (CheckBox)mContentView.findViewById(R.id.show_options); 137 mOptionsCheck.setOnClickListener(this); 138 mOptionsContainer = mContentView.findViewById(R.id.options); 139 mDisallowAssist = (CheckBox)mContentView.findViewById(R.id.disallow_structure); 140 mDisallowAssist.setOnClickListener(this); 141 mDisallowScreenshot = (CheckBox)mContentView.findViewById(R.id.disallow_screenshot); 142 mDisallowScreenshot.setOnClickListener(this); 143 mOptionsText = (TextView)mContentView.findViewById(R.id.options_text); 144 mConfirmButton = (Button)mContentView.findViewById(R.id.confirm); 145 mConfirmButton.setOnClickListener(this); 146 mCompleteButton = (Button)mContentView.findViewById(R.id.complete); 147 mCompleteButton.setOnClickListener(this); 148 mAbortButton = (Button)mContentView.findViewById(R.id.abort); 149 mAbortButton.setOnClickListener(this); 150 refreshOptions(); 151 return mContentView; 152 } 153 refreshOptions()154 void refreshOptions() { 155 if (mOptionsContainer != null) { 156 if (mOptionsCheck.isChecked()) { 157 mOptionsContainer.setVisibility(View.VISIBLE); 158 int flags = getDisabledShowContext(); 159 mDisallowAssist.setChecked((flags & SHOW_WITH_ASSIST) != 0); 160 mDisallowScreenshot.setChecked((flags & SHOW_WITH_SCREENSHOT) != 0); 161 int disabled = getUserDisabledShowContext(); 162 mOptionsText.setText("Disabled: 0x" + Integer.toHexString(disabled)); 163 } else { 164 mOptionsContainer.setVisibility(View.GONE); 165 } 166 } 167 } 168 onHandleAssist(Bundle assistBundle)169 public void onHandleAssist(Bundle assistBundle) { 170 } 171 172 @Override onHandleAssist(Bundle data, AssistStructure structure, AssistContent content)173 public void onHandleAssist(Bundle data, AssistStructure structure, AssistContent content) { 174 mAssistStructure = structure; 175 if (mAssistStructure != null) { 176 if (mAssistVisualizer != null) { 177 mAssistVisualizer.setAssistStructure(mAssistStructure); 178 } 179 } else { 180 if (mAssistVisualizer != null) { 181 mAssistVisualizer.clearAssistData(); 182 } 183 } 184 if (content != null) { 185 Log.i(TAG, "Assist intent: " + content.getIntent()); 186 Log.i(TAG, "Assist clipdata: " + content.getClipData()); 187 } 188 if (data != null) { 189 Uri referrer = data.getParcelable(Intent.EXTRA_REFERRER); 190 if (referrer != null) { 191 Log.i(TAG, "Referrer: " + referrer); 192 } 193 } 194 } 195 196 @Override onHandleAssistSecondary(final Bundle data, final AssistStructure structure, final AssistContent content, int index, int count)197 public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure, 198 final AssistContent content, int index, int count) { 199 Log.i(TAG, "Got secondary activity assist data " + index + " of " + count); 200 Log.i(TAG, "Showing assist structure after a few seconds..."); 201 mContentView.postDelayed(new Runnable() { 202 public void run() { 203 onHandleAssist(data, structure, content); 204 } 205 }, 2000 * index); 206 } 207 208 @Override onHandleScreenshot(Bitmap screenshot)209 public void onHandleScreenshot(Bitmap screenshot) { 210 if (screenshot != null) { 211 mScreenshot.setImageBitmap(screenshot); 212 mScreenshot.setAdjustViewBounds(true); 213 mScreenshot.setMaxWidth(screenshot.getWidth() / 3); 214 mScreenshot.setMaxHeight(screenshot.getHeight() / 3); 215 mFullScreenshot.setImageBitmap(screenshot); 216 } else { 217 mScreenshot.setImageDrawable(null); 218 mFullScreenshot.setImageDrawable(null); 219 } 220 } 221 updateState()222 void updateState() { 223 if (mState == STATE_IDLE) { 224 mTopContent.setVisibility(View.VISIBLE); 225 mBottomContent.setVisibility(View.GONE); 226 mAssistVisualizer.setVisibility(View.VISIBLE); 227 } else if (mState == STATE_DONE) { 228 mTopContent.setVisibility(View.GONE); 229 mBottomContent.setVisibility(View.GONE); 230 mAssistVisualizer.setVisibility(View.GONE); 231 } else { 232 mTopContent.setVisibility(View.GONE); 233 mBottomContent.setVisibility(View.VISIBLE); 234 mAssistVisualizer.setVisibility(View.GONE); 235 } 236 mStartButton.setEnabled(mState == STATE_IDLE); 237 mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_PICK_OPTION 238 || mState == STATE_COMMAND); 239 mAbortButton.setEnabled(mState == STATE_ABORT_VOICE); 240 mCompleteButton.setEnabled(mState == STATE_COMPLETE_VOICE); 241 } 242 onClick(View v)243 public void onClick(View v) { 244 if (v == mTreeButton) { 245 if (mAssistVisualizer != null) { 246 mAssistVisualizer.logTree(); 247 } 248 } else if (v == mTextButton) { 249 if (mAssistVisualizer != null) { 250 mAssistVisualizer.logText(); 251 } 252 } else if (v == mOptionsCheck) { 253 refreshOptions(); 254 } else if (v == mDisallowAssist) { 255 int flags = getDisabledShowContext(); 256 if (mDisallowAssist.isChecked()) { 257 flags |= SHOW_WITH_ASSIST; 258 } else { 259 flags &= ~SHOW_WITH_ASSIST; 260 } 261 setDisabledShowContext(flags); 262 } else if (v == mDisallowScreenshot) { 263 int flags = getDisabledShowContext(); 264 if (mDisallowScreenshot.isChecked()) { 265 flags |= SHOW_WITH_SCREENSHOT; 266 } else { 267 flags &= ~SHOW_WITH_SCREENSHOT; 268 } 269 setDisabledShowContext(flags); 270 } else if (v == mStartButton) { 271 mState = STATE_LAUNCHING; 272 updateState(); 273 startVoiceActivity(mStartIntent); 274 } else if (v == mConfirmButton) { 275 if (mPendingRequest instanceof ConfirmationRequest) { 276 ((ConfirmationRequest)mPendingRequest).sendConfirmationResult(true, null); 277 mPendingRequest = null; 278 mState = STATE_LAUNCHING; 279 } else if (mPendingRequest instanceof PickOptionRequest) { 280 PickOptionRequest pick = (PickOptionRequest)mPendingRequest; 281 int numReturn = mPendingOptions.length/2; 282 if (numReturn <= 0) { 283 numReturn = 1; 284 } 285 VoiceInteractor.PickOptionRequest.Option[] picked 286 = new VoiceInteractor.PickOptionRequest.Option[numReturn]; 287 for (int i=0; i<picked.length; i++) { 288 picked[i] = mPendingOptions[i*2]; 289 } 290 mPendingOptions = picked; 291 if (picked.length <= 1) { 292 pick.sendPickOptionResult(picked, null); 293 mPendingRequest = null; 294 mState = STATE_LAUNCHING; 295 } else { 296 pick.sendIntermediatePickOptionResult(picked, null); 297 updatePickText(); 298 } 299 } else if (mPendingRequest instanceof CommandRequest) { 300 Bundle result = new Bundle(); 301 result.putString("key", "a result!"); 302 ((CommandRequest)mPendingRequest).sendResult(result); 303 mPendingRequest = null; 304 mState = STATE_LAUNCHING; 305 } 306 } else if (v == mAbortButton && mPendingRequest instanceof AbortVoiceRequest) { 307 ((AbortVoiceRequest)mPendingRequest).sendAbortResult(null); 308 mPendingRequest = null; 309 } else if (v == mCompleteButton && mPendingRequest instanceof CompleteVoiceRequest) { 310 ((CompleteVoiceRequest)mPendingRequest).sendCompleteResult(null); 311 mPendingRequest = null; 312 } else if (v == mScreenshot) { 313 if (mFullScreenshot.getVisibility() != View.VISIBLE) { 314 mFullScreenshot.setVisibility(View.VISIBLE); 315 } else { 316 mFullScreenshot.setVisibility(View.INVISIBLE); 317 } 318 } 319 updateState(); 320 } 321 322 @Override onComputeInsets(Insets outInsets)323 public void onComputeInsets(Insets outInsets) { 324 super.onComputeInsets(outInsets); 325 if (mState != STATE_IDLE) { 326 outInsets.contentInsets.top = mBottomContent.getTop(); 327 outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; 328 } else if ((mShowFlags & SHOW_SOURCE_ACTIVITY) != 0) { 329 outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; 330 } 331 } 332 333 @Override onTaskStarted(Intent intent, int taskId)334 public void onTaskStarted(Intent intent, int taskId) { 335 super.onTaskStarted(intent, taskId); 336 mCurrentTask = taskId; 337 } 338 339 @Override onTaskFinished(Intent intent, int taskId)340 public void onTaskFinished(Intent intent, int taskId) { 341 super.onTaskFinished(intent, taskId); 342 if (mCurrentTask == taskId) { 343 mCurrentTask = -1; 344 } 345 } 346 347 @Override onLockscreenShown()348 public void onLockscreenShown() { 349 if (mCurrentTask < 0) { 350 hide(); 351 } 352 } 353 354 @Override onGetSupportedCommands(String[] commands)355 public boolean[] onGetSupportedCommands(String[] commands) { 356 boolean[] res = new boolean[commands.length]; 357 for (int i=0; i<commands.length; i++) { 358 if ("com.android.test.voiceinteraction.COMMAND".equals(commands[i])) { 359 res[i] = true; 360 } 361 } 362 return res; 363 } 364 setPrompt(VoiceInteractor.Prompt prompt)365 void setPrompt(VoiceInteractor.Prompt prompt) { 366 if (prompt == null) { 367 mText.setText("(null)"); 368 mPendingPrompt = ""; 369 } else { 370 mText.setText(prompt.getVisualPrompt()); 371 mPendingPrompt = prompt.getVisualPrompt(); 372 } 373 } 374 375 @Override onRequestConfirmation(ConfirmationRequest request)376 public void onRequestConfirmation(ConfirmationRequest request) { 377 Log.i(TAG, "onConfirm: prompt=" + request.getVoicePrompt() + " extras=" 378 + request.getExtras()); 379 setPrompt(request.getVoicePrompt()); 380 mConfirmButton.setText("Confirm"); 381 mPendingRequest = request; 382 mState = STATE_CONFIRM; 383 updateState(); 384 } 385 386 @Override onRequestPickOption(PickOptionRequest request)387 public void onRequestPickOption(PickOptionRequest request) { 388 Log.i(TAG, "onPickOption: prompt=" + request.getVoicePrompt() + " options=" 389 + request.getOptions() + " extras=" + request.getExtras()); 390 mConfirmButton.setText("Pick Option"); 391 mPendingRequest = request; 392 setPrompt(request.getVoicePrompt()); 393 mPendingOptions = request.getOptions(); 394 mState = STATE_PICK_OPTION; 395 updatePickText(); 396 updateState(); 397 } 398 updatePickText()399 void updatePickText() { 400 StringBuilder sb = new StringBuilder(); 401 sb.append(mPendingPrompt); 402 sb.append(": "); 403 for (int i=0; i<mPendingOptions.length; i++) { 404 if (i > 0) { 405 sb.append(", "); 406 } 407 sb.append(mPendingOptions[i].getLabel()); 408 } 409 mText.setText(sb.toString()); 410 } 411 412 @Override onRequestCompleteVoice(CompleteVoiceRequest request)413 public void onRequestCompleteVoice(CompleteVoiceRequest request) { 414 Log.i(TAG, "onCompleteVoice: message=" + request.getVoicePrompt() + " extras=" 415 + request.getExtras()); 416 setPrompt(request.getVoicePrompt()); 417 mPendingRequest = request; 418 mState = STATE_COMPLETE_VOICE; 419 updateState(); 420 } 421 422 @Override onRequestAbortVoice(AbortVoiceRequest request)423 public void onRequestAbortVoice(AbortVoiceRequest request) { 424 Log.i(TAG, "onAbortVoice: message=" + request.getVoicePrompt() + " extras=" 425 + request.getExtras()); 426 setPrompt(request.getVoicePrompt()); 427 mPendingRequest = request; 428 mState = STATE_ABORT_VOICE; 429 updateState(); 430 } 431 432 @Override onRequestCommand(CommandRequest request)433 public void onRequestCommand(CommandRequest request) { 434 Bundle extras = request.getExtras(); 435 if (extras != null) { 436 extras.getString("arg"); 437 } 438 Log.i(TAG, "onCommand: command=" + request.getCommand() + " extras=" + extras); 439 mText.setText("Command: " + request.getCommand() + ", " + extras); 440 mConfirmButton.setText("Finish Command"); 441 mPendingRequest = request; 442 mState = STATE_COMMAND; 443 updateState(); 444 } 445 446 @Override onCancelRequest(Request request)447 public void onCancelRequest(Request request) { 448 Log.i(TAG, "onCancel"); 449 if (mPendingRequest == request) { 450 mPendingRequest = null; 451 mState = STATE_LAUNCHING; 452 updateState(); 453 } 454 request.cancel(); 455 } 456 } 457