1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.voiceinteraction.service; 18 19 import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; 20 import static android.Manifest.permission.RECORD_AUDIO; 21 22 import static com.android.compatibility.common.util.SystemUtil.runWithShellPermissionIdentity; 23 24 import android.Manifest; 25 import android.app.UiAutomation; 26 import android.content.Intent; 27 import android.hardware.soundtrigger.SoundTrigger; 28 import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; 29 import android.media.AudioFormat; 30 import android.os.ParcelFileDescriptor; 31 import android.os.Parcelable; 32 import android.os.PersistableBundle; 33 import android.os.SharedMemory; 34 import android.service.voice.AlwaysOnHotwordDetector; 35 import android.service.voice.HotwordDetectionService; 36 import android.service.voice.HotwordDetector; 37 import android.service.voice.HotwordRejectedResult; 38 import android.service.voice.VoiceInteractionService; 39 import android.system.ErrnoException; 40 import android.util.Log; 41 import android.voiceinteraction.common.Utils; 42 43 import androidx.annotation.NonNull; 44 import androidx.test.platform.app.InstrumentationRegistry; 45 46 import com.google.common.collect.ImmutableList; 47 48 import java.io.IOException; 49 import java.io.OutputStream; 50 import java.nio.ByteBuffer; 51 import java.util.Locale; 52 53 /** 54 * This service included a basic HotwordDetectionService for testing. 55 */ 56 public class BasicVoiceInteractionService extends VoiceInteractionService { 57 // TODO: (b/182236586) Refactor the voice interaction service logic 58 static final String TAG = "BasicVoiceInteractionService"; 59 60 public static String KEY_FAKE_DATA = "fakeData"; 61 public static String VALUE_FAKE_DATA = "fakeData"; 62 public static byte[] FAKE_BYTE_ARRAY_DATA = new byte[]{1, 2, 3}; 63 public static byte[] FAKE_HOTWORD_AUDIO_DATA = 64 new byte[]{'h', 'o', 't', 'w', 'o', 'r', 'd', '!'}; 65 66 private boolean mReady = false; 67 private AlwaysOnHotwordDetector mAlwaysOnHotwordDetector = null; 68 private HotwordDetector mSoftwareHotwordDetector = null; 69 private ParcelFileDescriptor[] mTempParcelFileDescriptor = null; 70 71 @Override onReady()72 public void onReady() { 73 super.onReady(); 74 mReady = true; 75 } 76 77 @Override onStartCommand(Intent intent, int flags, int startId)78 public int onStartCommand(Intent intent, int flags, int startId) { 79 Log.i(TAG, "onStartCommand received"); 80 81 if (intent == null || !mReady) { 82 Log.wtf(TAG, "Can't start because either intent is null or onReady() " 83 + "is not called yet. intent = " + intent + ", mReady = " + mReady); 84 return START_NOT_STICKY; 85 } 86 87 UiAutomation uiAutomation = InstrumentationRegistry.getInstrumentation().getUiAutomation(); 88 // Drop any identity adopted earlier. 89 uiAutomation.dropShellPermissionIdentity(); 90 91 final int testEvent = intent.getIntExtra(Utils.KEY_TEST_EVENT, -1); 92 Log.i(TAG, "testEvent = " + testEvent); 93 94 try { 95 if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_TEST) { 96 runWithShellPermissionIdentity(() -> { 97 mAlwaysOnHotwordDetector = callCreateAlwaysOnHotwordDetector(); 98 }, Manifest.permission.MANAGE_HOTWORD_DETECTION); 99 } else if (testEvent == Utils.VIS_WITHOUT_MANAGE_HOTWORD_DETECTION_PERMISSION_TEST) { 100 runWithShellPermissionIdentity(() -> callCreateAlwaysOnHotwordDetector(), 101 Manifest.permission.BIND_HOTWORD_DETECTION_SERVICE); 102 } else if (testEvent == Utils.VIS_HOLD_BIND_HOTWORD_DETECTION_PERMISSION_TEST) { 103 runWithShellPermissionIdentity(() -> callCreateAlwaysOnHotwordDetector()); 104 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_DSP_ONDETECT_TEST) { 105 // need to retain the identity until the callback is triggered 106 uiAutomation.adoptShellPermissionIdentity(RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD); 107 if (mAlwaysOnHotwordDetector != null) { 108 mAlwaysOnHotwordDetector.triggerHardwareRecognitionEventForTest(/* status */ 0, 109 /* soundModelHandle */ 100, /* captureAvailable */ true, 110 /* captureSession */ 101, /* captureDelayMs */ 1000, 111 /* capturePreambleMs */ 1001, /* triggerInData */ true, 112 createFakeAudioFormat(), new byte[1024], 113 ImmutableList.of(new KeyphraseRecognitionExtra( 114 MainHotwordDetectionService.DEFAULT_PHRASE_ID, 115 SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, 100))); 116 } 117 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_DSP_ONREJECT_TEST) { 118 // need to retain the identity until the callback is triggered 119 uiAutomation.adoptShellPermissionIdentity(RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD); 120 if (mAlwaysOnHotwordDetector != null) { 121 mAlwaysOnHotwordDetector.triggerHardwareRecognitionEventForTest(/* status */ 122 0, 123 /* soundModelHandle */ 100, /* captureAvailable */ true, 124 /* captureSession */ 101, /* captureDelayMs */ 1000, 125 /* capturePreambleMs */ 1001, /* triggerInData */ true, 126 createFakeAudioFormat(), null, 127 ImmutableList.of(new KeyphraseRecognitionExtra( 128 MainHotwordDetectionService.DEFAULT_PHRASE_ID, 129 SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, 100))); 130 } 131 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_EXTERNAL_SOURCE_ONDETECT_TEST) { 132 uiAutomation.adoptShellPermissionIdentity(RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD); 133 if (mAlwaysOnHotwordDetector != null) { 134 ParcelFileDescriptor audioStream = createFakeAudioStream(); 135 if (audioStream != null) { 136 mAlwaysOnHotwordDetector.startRecognition( 137 audioStream, 138 createFakeAudioFormat(), 139 createFakePersistableBundleData()); 140 } 141 } 142 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_FROM_SOFTWARE_TRIGGER_TEST) { 143 runWithShellPermissionIdentity(() -> { 144 mSoftwareHotwordDetector = callCreateSoftwareHotwordDetector(); 145 }, Manifest.permission.MANAGE_HOTWORD_DETECTION); 146 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_MIC_ONDETECT_TEST) { 147 uiAutomation.adoptShellPermissionIdentity(RECORD_AUDIO, CAPTURE_AUDIO_HOTWORD); 148 if (mSoftwareHotwordDetector != null) { 149 mSoftwareHotwordDetector.startRecognition(); 150 } 151 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_CALL_STOP_RECOGNITION) { 152 if (mSoftwareHotwordDetector != null) { 153 mSoftwareHotwordDetector.stopRecognition(); 154 } 155 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_PROCESS_DIED_TEST) { 156 runWithShellPermissionIdentity(() -> { 157 if (mAlwaysOnHotwordDetector != null) { 158 PersistableBundle persistableBundle = new PersistableBundle(); 159 persistableBundle.putInt(Utils.KEY_TEST_SCENARIO, 160 Utils.HOTWORD_DETECTION_SERVICE_ON_UPDATE_STATE_CRASH); 161 mAlwaysOnHotwordDetector.updateState( 162 persistableBundle, 163 createFakeSharedMemoryData()); 164 } 165 }, Manifest.permission.MANAGE_HOTWORD_DETECTION); 166 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_DSP_DESTROY_DETECTOR) { 167 if (mAlwaysOnHotwordDetector != null) { 168 Log.i(TAG, "destroying AlwaysOnHotwordDetector"); 169 mAlwaysOnHotwordDetector.destroy(); 170 broadcastIntentWithResult( 171 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 172 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_SUCCESS); 173 } 174 } else if (testEvent == Utils.HOTWORD_DETECTION_SERVICE_SOFTWARE_DESTROY_DETECTOR) { 175 if (mSoftwareHotwordDetector != null) { 176 Log.i(TAG, "destroying SoftwareHotwordDetector"); 177 mSoftwareHotwordDetector.destroy(); 178 broadcastIntentWithResult( 179 Utils.HOTWORD_DETECTION_SERVICE_SOFTWARE_TRIGGER_RESULT_INTENT, 180 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_SUCCESS); 181 } 182 } 183 } catch (IllegalStateException e) { 184 Log.w(TAG, "performing testEvent: " + testEvent + ", exception: " + e); 185 broadcastIntentWithResult( 186 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 187 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_ILLEGAL_STATE_EXCEPTION); 188 } 189 190 return START_NOT_STICKY; 191 } 192 193 @Override onDestroy()194 public void onDestroy() { 195 super.onDestroy(); 196 closeFakeAudioStream(); 197 InstrumentationRegistry.getInstrumentation().getUiAutomation() 198 .dropShellPermissionIdentity(); 199 } 200 callCreateAlwaysOnHotwordDetector()201 private AlwaysOnHotwordDetector callCreateAlwaysOnHotwordDetector() { 202 Log.i(TAG, "callCreateAlwaysOnHotwordDetector()"); 203 try { 204 return createAlwaysOnHotwordDetector(/* keyphrase */ "Hello Android", 205 Locale.forLanguageTag("en-US"), 206 createFakePersistableBundleData(), 207 createFakeSharedMemoryData(), 208 new AlwaysOnHotwordDetector.Callback() { 209 @Override 210 public void onAvailabilityChanged(int status) { 211 Log.i(TAG, "onAvailabilityChanged(" + status + ")"); 212 } 213 214 @Override 215 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 216 Log.i(TAG, "onDetected"); 217 broadcastIntentWithResult( 218 Utils.HOTWORD_DETECTION_SERVICE_ONDETECT_RESULT_INTENT, 219 new EventPayloadParcelable(eventPayload)); 220 } 221 222 @Override 223 public void onRejected(@NonNull HotwordRejectedResult result) { 224 super.onRejected(result); 225 Log.i(TAG, "onRejected"); 226 broadcastIntentWithResult( 227 Utils.HOTWORD_DETECTION_SERVICE_ONDETECT_RESULT_INTENT, 228 result); 229 } 230 231 @Override 232 public void onError() { 233 Log.i(TAG, "onError"); 234 broadcastIntentWithResult( 235 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 236 Utils.HOTWORD_DETECTION_SERVICE_GET_ERROR); 237 } 238 239 @Override 240 public void onRecognitionPaused() { 241 Log.i(TAG, "onRecognitionPaused"); 242 } 243 244 @Override 245 public void onRecognitionResumed() { 246 Log.i(TAG, "onRecognitionResumed"); 247 } 248 249 @Override 250 public void onHotwordDetectionServiceInitialized(int status) { 251 super.onHotwordDetectionServiceInitialized(status); 252 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 253 if (status != HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS) { 254 return; 255 } 256 broadcastIntentWithResult( 257 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 258 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_SUCCESS); 259 } 260 261 @Override 262 public void onHotwordDetectionServiceRestarted() { 263 super.onHotwordDetectionServiceRestarted(); 264 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 265 } 266 }); 267 } catch (IllegalStateException e) { 268 Log.w(TAG, "callCreateAlwaysOnHotwordDetector() exception: " + e); 269 broadcastIntentWithResult( 270 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 271 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_ILLEGAL_STATE_EXCEPTION); 272 } catch (SecurityException e) { 273 Log.w(TAG, "callCreateAlwaysOnHotwordDetector() exception: " + e); 274 broadcastIntentWithResult( 275 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_RESULT_INTENT, 276 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_SECURITY_EXCEPTION); 277 } 278 return null; 279 } 280 281 private HotwordDetector callCreateSoftwareHotwordDetector() { 282 Log.i(TAG, "callCreateSoftwareHotwordDetector()"); 283 try { 284 return createHotwordDetector( 285 createFakePersistableBundleData(), 286 createFakeSharedMemoryData(), 287 new HotwordDetector.Callback() { 288 @Override 289 public void onDetected(AlwaysOnHotwordDetector.EventPayload eventPayload) { 290 Log.i(TAG, "onDetected"); 291 broadcastIntentWithResult( 292 Utils.HOTWORD_DETECTION_SERVICE_ONDETECT_RESULT_INTENT, 293 new EventPayloadParcelable(eventPayload)); 294 } 295 296 @Override 297 public void onError() { 298 Log.i(TAG, "onError"); 299 broadcastIntentWithResult( 300 Utils.HOTWORD_DETECTION_SERVICE_SOFTWARE_TRIGGER_RESULT_INTENT, 301 Utils.HOTWORD_DETECTION_SERVICE_GET_ERROR); 302 } 303 304 @Override 305 public void onRecognitionPaused() { 306 Log.i(TAG, "onRecognitionPaused"); 307 } 308 309 @Override 310 public void onRecognitionResumed() { 311 Log.i(TAG, "onRecognitionResumed"); 312 } 313 314 @Override 315 public void onRejected(HotwordRejectedResult result) { 316 Log.i(TAG, "onRejected"); 317 } 318 319 @Override 320 public void onHotwordDetectionServiceInitialized(int status) { 321 Log.i(TAG, "onHotwordDetectionServiceInitialized status = " + status); 322 if (status != HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS) { 323 return; 324 } 325 broadcastIntentWithResult( 326 Utils.HOTWORD_DETECTION_SERVICE_SOFTWARE_TRIGGER_RESULT_INTENT, 327 Utils.HOTWORD_DETECTION_SERVICE_TRIGGER_SUCCESS); 328 } 329 330 @Override 331 public void onHotwordDetectionServiceRestarted() { 332 Log.i(TAG, "onHotwordDetectionServiceRestarted"); 333 } 334 }); 335 } catch (Exception e) { 336 Log.w(TAG, "callCreateSoftwareHotwordDetector() exception: " + e); 337 } 338 return null; 339 } 340 341 private void broadcastIntentWithResult(String intentName, int result) { 342 Intent intent = new Intent(intentName) 343 .addFlags(Intent.FLAG_RECEIVER_FOREGROUND | Intent.FLAG_RECEIVER_REGISTERED_ONLY) 344 .putExtra(Utils.KEY_TEST_RESULT, result); 345 Log.d(TAG, "broadcast intent = " + intent + ", result = " + result); 346 sendBroadcast(intent); 347 } 348 349 private void broadcastIntentWithResult(String intentName, Parcelable result) { 350 Intent intent = new Intent(intentName) 351 .addFlags(Intent.FLAG_RECEIVER_FOREGROUND | Intent.FLAG_RECEIVER_REGISTERED_ONLY) 352 .putExtra(Utils.KEY_TEST_RESULT, result); 353 Log.d(TAG, "broadcast intent = " + intent + ", result = " + result); 354 sendBroadcast(intent); 355 } 356 357 private SharedMemory createFakeSharedMemoryData() { 358 try { 359 SharedMemory sharedMemory = SharedMemory.create("SharedMemory", 3); 360 ByteBuffer byteBuffer = sharedMemory.mapReadWrite(); 361 byteBuffer.put(FAKE_BYTE_ARRAY_DATA); 362 return sharedMemory; 363 } catch (ErrnoException e) { 364 Log.w(TAG, "createFakeSharedMemoryData ErrnoException : " + e); 365 throw new RuntimeException(e.getMessage()); 366 } 367 } 368 369 private PersistableBundle createFakePersistableBundleData() { 370 // TODO : Add more data for testing 371 PersistableBundle persistableBundle = new PersistableBundle(); 372 persistableBundle.putString(KEY_FAKE_DATA, VALUE_FAKE_DATA); 373 return persistableBundle; 374 } 375 376 private AudioFormat createFakeAudioFormat() { 377 return new AudioFormat.Builder() 378 .setSampleRate(32000) 379 .setEncoding(AudioFormat.ENCODING_PCM_16BIT) 380 .setChannelMask(AudioFormat.CHANNEL_IN_MONO).build(); 381 } 382 383 private ParcelFileDescriptor createFakeAudioStream() { 384 try { 385 mTempParcelFileDescriptor = ParcelFileDescriptor.createPipe(); 386 try (OutputStream fos = 387 new ParcelFileDescriptor.AutoCloseOutputStream( 388 mTempParcelFileDescriptor[1])) { 389 fos.write(FAKE_HOTWORD_AUDIO_DATA, 0, 8); 390 } catch (IOException e) { 391 Log.w(TAG, "Failed to pipe audio data : ", e); 392 return null; 393 } 394 return mTempParcelFileDescriptor[0]; 395 } catch (IOException e) { 396 Log.w(TAG, "Failed to create a pipe : " + e); 397 } 398 return null; 399 } 400 401 private void closeFakeAudioStream() { 402 if (mTempParcelFileDescriptor != null) { 403 try { 404 mTempParcelFileDescriptor[0].close(); 405 mTempParcelFileDescriptor[1].close(); 406 } catch (IOException e) { 407 Log.w(TAG, "Failed closing : " + e); 408 } 409 mTempParcelFileDescriptor = null; 410 } 411 } 412 } 413