1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.test.voiceinteraction; 18 19 import android.app.ActivityManager; 20 import android.app.VoiceInteractor; 21 import android.app.assist.AssistContent; 22 import android.app.assist.AssistStructure; 23 import android.content.Context; 24 import android.content.Intent; 25 import android.graphics.Bitmap; 26 import android.net.Uri; 27 import android.os.Bundle; 28 import android.service.voice.VoiceInteractionSession; 29 import android.util.Log; 30 import android.view.View; 31 import android.widget.Button; 32 import android.widget.CheckBox; 33 import android.widget.ImageView; 34 import android.widget.TextView; 35 36 public class MainInteractionSession extends VoiceInteractionSession 37 implements View.OnClickListener { 38 static final String TAG = "MainInteractionSession"; 39 40 Intent mStartIntent; 41 View mContentView; 42 AssistVisualizer mAssistVisualizer; 43 View mTopContent; 44 View mBottomContent; 45 TextView mText; 46 Button mTreeButton; 47 Button mTextButton; 48 Button mStartButton; 49 CheckBox mOptionsCheck; 50 View mOptionsContainer; 51 CheckBox mDisallowAssist; 52 CheckBox mDisallowScreenshot; 53 TextView mOptionsText; 54 ImageView mScreenshot; 55 ImageView mFullScreenshot; 56 Button mConfirmButton; 57 Button mCompleteButton; 58 Button mAbortButton; 59 60 AssistStructure mAssistStructure; 61 62 static final int STATE_IDLE = 0; 63 static final int STATE_LAUNCHING = 1; 64 static final int STATE_CONFIRM = 2; 65 static final int STATE_PICK_OPTION = 3; 66 static final int STATE_COMMAND = 4; 67 static final int STATE_ABORT_VOICE = 5; 68 static final int STATE_COMPLETE_VOICE = 6; 69 static final int STATE_DONE = 7; 70 71 int mState = STATE_IDLE; 72 VoiceInteractor.PickOptionRequest.Option[] mPendingOptions; 73 CharSequence mPendingPrompt; 74 Request mPendingRequest; 75 int mCurrentTask = -1; 76 int mShowFlags; 77 MainInteractionSession(Context context)78 MainInteractionSession(Context context) { 79 super(context); 80 } 81 82 @Override onCreate()83 public void onCreate() { 84 super.onCreate(); 85 ActivityManager am = getContext().getSystemService(ActivityManager.class); 86 am.setWatchHeapLimit(40 * 1024 * 1024); 87 } 88 89 @Override onShow(Bundle args, int showFlags)90 public void onShow(Bundle args, int showFlags) { 91 super.onShow(args, showFlags); 92 mShowFlags = showFlags; 93 Log.i(TAG, "onShow: flags=0x" + Integer.toHexString(showFlags) + " args=" + args); 94 mState = STATE_IDLE; 95 mStartIntent = args != null ? (Intent)args.getParcelable("intent") : null; 96 if (mStartIntent == null) { 97 mStartIntent = new Intent(getContext(), TestInteractionActivity.class); 98 } 99 if (mAssistVisualizer != null) { 100 mAssistVisualizer.clearAssistData(); 101 } 102 onHandleScreenshot(null); 103 updateState(); 104 refreshOptions(); 105 } 106 107 @Override onHide()108 public void onHide() { 109 super.onHide(); 110 if (mAssistVisualizer != null) { 111 mAssistVisualizer.clearAssistData(); 112 } 113 mState = STATE_DONE; 114 updateState(); 115 } 116 117 @Override onCreateContentView()118 public View onCreateContentView() { 119 mContentView = getLayoutInflater().inflate(R.layout.voice_interaction_session, null); 120 mAssistVisualizer = (AssistVisualizer)mContentView.findViewById(R.id.assist_visualizer); 121 if (mAssistStructure != null) { 122 mAssistVisualizer.setAssistStructure(mAssistStructure); 123 } 124 mTopContent = mContentView.findViewById(R.id.top_content); 125 mBottomContent = mContentView.findViewById(R.id.bottom_content); 126 mText = (TextView)mContentView.findViewById(R.id.text); 127 mTreeButton = (Button)mContentView.findViewById(R.id.do_tree); 128 mTreeButton.setOnClickListener(this); 129 mTextButton = (Button)mContentView.findViewById(R.id.do_text); 130 mTextButton.setOnClickListener(this); 131 mStartButton = (Button)mContentView.findViewById(R.id.start); 132 mStartButton.setOnClickListener(this); 133 mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot); 134 mScreenshot.setOnClickListener(this); 135 mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot); 136 mOptionsCheck = (CheckBox)mContentView.findViewById(R.id.show_options); 137 mOptionsCheck.setOnClickListener(this); 138 mOptionsContainer = mContentView.findViewById(R.id.options); 139 mDisallowAssist = (CheckBox)mContentView.findViewById(R.id.disallow_structure); 140 mDisallowAssist.setOnClickListener(this); 141 mDisallowScreenshot = (CheckBox)mContentView.findViewById(R.id.disallow_screenshot); 142 mDisallowScreenshot.setOnClickListener(this); 143 mOptionsText = (TextView)mContentView.findViewById(R.id.options_text); 144 mConfirmButton = (Button)mContentView.findViewById(R.id.confirm); 145 mConfirmButton.setOnClickListener(this); 146 mCompleteButton = (Button)mContentView.findViewById(R.id.complete); 147 mCompleteButton.setOnClickListener(this); 148 mAbortButton = (Button)mContentView.findViewById(R.id.abort); 149 mAbortButton.setOnClickListener(this); 150 refreshOptions(); 151 return mContentView; 152 } 153 refreshOptions()154 void refreshOptions() { 155 if (mOptionsContainer != null) { 156 if (mOptionsCheck.isChecked()) { 157 mOptionsContainer.setVisibility(View.VISIBLE); 158 int flags = getDisabledShowContext(); 159 mDisallowAssist.setChecked((flags & SHOW_WITH_ASSIST) != 0); 160 mDisallowScreenshot.setChecked((flags & SHOW_WITH_SCREENSHOT) != 0); 161 int disabled = getUserDisabledShowContext(); 162 mOptionsText.setText("Disabled: 0x" + Integer.toHexString(disabled)); 163 } else { 164 mOptionsContainer.setVisibility(View.GONE); 165 } 166 } 167 } 168 onHandleAssist(Bundle assistBundle)169 public void onHandleAssist(Bundle assistBundle) { 170 } 171 172 @Override onHandleAssist(Bundle data, AssistStructure structure, AssistContent content)173 public void onHandleAssist(Bundle data, AssistStructure structure, AssistContent content) { 174 mAssistStructure = structure; 175 if (mAssistVisualizer != null) { 176 if (mAssistStructure != null) { 177 mAssistVisualizer.setAssistStructure(mAssistStructure); 178 } else { 179 mAssistVisualizer.clearAssistData(); 180 } 181 } 182 if (content != null) { 183 Log.i(TAG, "Assist intent: " + content.getIntent()); 184 Log.i(TAG, "Assist clipdata: " + content.getClipData()); 185 } 186 if (data != null) { 187 Uri referrer = data.getParcelable(Intent.EXTRA_REFERRER); 188 if (referrer != null) { 189 Log.i(TAG, "Referrer: " + referrer); 190 } 191 } 192 } 193 194 @Override onHandleAssistSecondary(final Bundle data, final AssistStructure structure, final AssistContent content, int index, int count)195 public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure, 196 final AssistContent content, int index, int count) { 197 Log.i(TAG, "Got secondary activity assist data " + index + " of " + count); 198 Log.i(TAG, "Showing assist structure after a few seconds..."); 199 mContentView.postDelayed(new Runnable() { 200 public void run() { 201 onHandleAssist(data, structure, content); 202 } 203 }, 2000 * index); 204 } 205 206 @Override onHandleScreenshot(Bitmap screenshot)207 public void onHandleScreenshot(Bitmap screenshot) { 208 if (mScreenshot != null) { 209 if (screenshot != null) { 210 mScreenshot.setImageBitmap(screenshot); 211 mScreenshot.setAdjustViewBounds(true); 212 mScreenshot.setMaxWidth(screenshot.getWidth() / 3); 213 mScreenshot.setMaxHeight(screenshot.getHeight() / 3); 214 mFullScreenshot.setImageBitmap(screenshot); 215 } else { 216 mScreenshot.setImageDrawable(null); 217 mFullScreenshot.setImageDrawable(null); 218 } 219 } 220 } 221 updateState()222 void updateState() { 223 if (mTopContent == null) { 224 return; 225 } 226 if (mState == STATE_IDLE) { 227 mTopContent.setVisibility(View.VISIBLE); 228 mBottomContent.setVisibility(View.GONE); 229 mAssistVisualizer.setVisibility(View.VISIBLE); 230 } else if (mState == STATE_DONE) { 231 mTopContent.setVisibility(View.GONE); 232 mBottomContent.setVisibility(View.GONE); 233 mAssistVisualizer.setVisibility(View.GONE); 234 } else { 235 mTopContent.setVisibility(View.GONE); 236 mBottomContent.setVisibility(View.VISIBLE); 237 mAssistVisualizer.setVisibility(View.GONE); 238 } 239 mStartButton.setEnabled(mState == STATE_IDLE); 240 mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_PICK_OPTION 241 || mState == STATE_COMMAND); 242 mAbortButton.setEnabled(mState == STATE_ABORT_VOICE); 243 mCompleteButton.setEnabled(mState == STATE_COMPLETE_VOICE); 244 } 245 onClick(View v)246 public void onClick(View v) { 247 if (v == mTreeButton) { 248 if (mAssistVisualizer != null) { 249 mAssistVisualizer.logTree(); 250 } 251 } else if (v == mTextButton) { 252 if (mAssistVisualizer != null) { 253 mAssistVisualizer.logText(); 254 } 255 } else if (v == mOptionsCheck) { 256 refreshOptions(); 257 } else if (v == mDisallowAssist) { 258 int flags = getDisabledShowContext(); 259 if (mDisallowAssist.isChecked()) { 260 flags |= SHOW_WITH_ASSIST; 261 } else { 262 flags &= ~SHOW_WITH_ASSIST; 263 } 264 setDisabledShowContext(flags); 265 } else if (v == mDisallowScreenshot) { 266 int flags = getDisabledShowContext(); 267 if (mDisallowScreenshot.isChecked()) { 268 flags |= SHOW_WITH_SCREENSHOT; 269 } else { 270 flags &= ~SHOW_WITH_SCREENSHOT; 271 } 272 setDisabledShowContext(flags); 273 } else if (v == mStartButton) { 274 mState = STATE_LAUNCHING; 275 updateState(); 276 startVoiceActivity(mStartIntent); 277 } else if (v == mConfirmButton) { 278 if (mPendingRequest instanceof ConfirmationRequest) { 279 ((ConfirmationRequest)mPendingRequest).sendConfirmationResult(true, null); 280 mPendingRequest = null; 281 mState = STATE_LAUNCHING; 282 } else if (mPendingRequest instanceof PickOptionRequest) { 283 PickOptionRequest pick = (PickOptionRequest)mPendingRequest; 284 int numReturn = mPendingOptions.length/2; 285 if (numReturn <= 0) { 286 numReturn = 1; 287 } 288 VoiceInteractor.PickOptionRequest.Option[] picked 289 = new VoiceInteractor.PickOptionRequest.Option[numReturn]; 290 for (int i=0; i<picked.length; i++) { 291 picked[i] = mPendingOptions[i*2]; 292 } 293 mPendingOptions = picked; 294 if (picked.length <= 1) { 295 pick.sendPickOptionResult(picked, null); 296 mPendingRequest = null; 297 mState = STATE_LAUNCHING; 298 } else { 299 pick.sendIntermediatePickOptionResult(picked, null); 300 updatePickText(); 301 } 302 } else if (mPendingRequest instanceof CommandRequest) { 303 Bundle result = new Bundle(); 304 result.putString("key", "a result!"); 305 ((CommandRequest)mPendingRequest).sendResult(result); 306 mPendingRequest = null; 307 mState = STATE_LAUNCHING; 308 } 309 } else if (v == mAbortButton && mPendingRequest instanceof AbortVoiceRequest) { 310 ((AbortVoiceRequest)mPendingRequest).sendAbortResult(null); 311 mPendingRequest = null; 312 } else if (v == mCompleteButton && mPendingRequest instanceof CompleteVoiceRequest) { 313 ((CompleteVoiceRequest)mPendingRequest).sendCompleteResult(null); 314 mPendingRequest = null; 315 } else if (v == mScreenshot) { 316 if (mFullScreenshot.getVisibility() != View.VISIBLE) { 317 mFullScreenshot.setVisibility(View.VISIBLE); 318 } else { 319 mFullScreenshot.setVisibility(View.INVISIBLE); 320 } 321 } 322 updateState(); 323 } 324 325 @Override onComputeInsets(Insets outInsets)326 public void onComputeInsets(Insets outInsets) { 327 super.onComputeInsets(outInsets); 328 if (mState != STATE_IDLE) { 329 outInsets.contentInsets.top = mBottomContent.getTop(); 330 outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; 331 } else if ((mShowFlags & SHOW_SOURCE_ACTIVITY) != 0) { 332 outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; 333 } 334 } 335 336 @Override onTaskStarted(Intent intent, int taskId)337 public void onTaskStarted(Intent intent, int taskId) { 338 super.onTaskStarted(intent, taskId); 339 mCurrentTask = taskId; 340 } 341 342 @Override onTaskFinished(Intent intent, int taskId)343 public void onTaskFinished(Intent intent, int taskId) { 344 super.onTaskFinished(intent, taskId); 345 if (mCurrentTask == taskId) { 346 mCurrentTask = -1; 347 } 348 } 349 350 @Override onLockscreenShown()351 public void onLockscreenShown() { 352 if (mCurrentTask < 0) { 353 hide(); 354 } 355 } 356 357 @Override onGetSupportedCommands(String[] commands)358 public boolean[] onGetSupportedCommands(String[] commands) { 359 boolean[] res = new boolean[commands.length]; 360 for (int i=0; i<commands.length; i++) { 361 if ("com.android.test.voiceinteraction.COMMAND".equals(commands[i])) { 362 res[i] = true; 363 } 364 } 365 return res; 366 } 367 setPrompt(VoiceInteractor.Prompt prompt)368 void setPrompt(VoiceInteractor.Prompt prompt) { 369 if (prompt == null) { 370 mText.setText("(null)"); 371 mPendingPrompt = ""; 372 } else { 373 mText.setText(prompt.getVisualPrompt()); 374 mPendingPrompt = prompt.getVisualPrompt(); 375 } 376 } 377 378 @Override onRequestConfirmation(ConfirmationRequest request)379 public void onRequestConfirmation(ConfirmationRequest request) { 380 Log.i(TAG, "onConfirm: prompt=" + request.getVoicePrompt() + " extras=" 381 + request.getExtras()); 382 setPrompt(request.getVoicePrompt()); 383 mConfirmButton.setText("Confirm"); 384 mPendingRequest = request; 385 mState = STATE_CONFIRM; 386 updateState(); 387 } 388 389 @Override onRequestPickOption(PickOptionRequest request)390 public void onRequestPickOption(PickOptionRequest request) { 391 Log.i(TAG, "onPickOption: prompt=" + request.getVoicePrompt() + " options=" 392 + request.getOptions() + " extras=" + request.getExtras()); 393 mConfirmButton.setText("Pick Option"); 394 mPendingRequest = request; 395 setPrompt(request.getVoicePrompt()); 396 mPendingOptions = request.getOptions(); 397 mState = STATE_PICK_OPTION; 398 updatePickText(); 399 updateState(); 400 } 401 updatePickText()402 void updatePickText() { 403 StringBuilder sb = new StringBuilder(); 404 sb.append(mPendingPrompt); 405 sb.append(": "); 406 for (int i=0; i<mPendingOptions.length; i++) { 407 if (i > 0) { 408 sb.append(", "); 409 } 410 sb.append(mPendingOptions[i].getLabel()); 411 } 412 mText.setText(sb.toString()); 413 } 414 415 @Override onRequestCompleteVoice(CompleteVoiceRequest request)416 public void onRequestCompleteVoice(CompleteVoiceRequest request) { 417 Log.i(TAG, "onCompleteVoice: message=" + request.getVoicePrompt() + " extras=" 418 + request.getExtras()); 419 setPrompt(request.getVoicePrompt()); 420 mPendingRequest = request; 421 mState = STATE_COMPLETE_VOICE; 422 updateState(); 423 } 424 425 @Override onRequestAbortVoice(AbortVoiceRequest request)426 public void onRequestAbortVoice(AbortVoiceRequest request) { 427 Log.i(TAG, "onAbortVoice: message=" + request.getVoicePrompt() + " extras=" 428 + request.getExtras()); 429 setPrompt(request.getVoicePrompt()); 430 mPendingRequest = request; 431 mState = STATE_ABORT_VOICE; 432 updateState(); 433 } 434 435 @Override onRequestCommand(CommandRequest request)436 public void onRequestCommand(CommandRequest request) { 437 Bundle extras = request.getExtras(); 438 if (extras != null) { 439 extras.getString("arg"); 440 } 441 Log.i(TAG, "onCommand: command=" + request.getCommand() + " extras=" + extras); 442 mText.setText("Command: " + request.getCommand() + ", " + extras); 443 mConfirmButton.setText("Finish Command"); 444 mPendingRequest = request; 445 mState = STATE_COMMAND; 446 updateState(); 447 } 448 449 @Override onCancelRequest(Request request)450 public void onCancelRequest(Request request) { 451 Log.i(TAG, "onCancel"); 452 if (mPendingRequest == request) { 453 mPendingRequest = null; 454 mState = STATE_LAUNCHING; 455 updateState(); 456 } 457 request.cancel(); 458 } 459 } 460