1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.example.android.voiceinteractor; 18 19 import android.graphics.ImageFormat; 20 import android.hardware.camera2.CameraAccessException; 21 import android.hardware.camera2.CameraCaptureSession; 22 import android.hardware.camera2.CameraCharacteristics; 23 import android.hardware.camera2.CameraDevice; 24 import android.hardware.camera2.CameraManager; 25 import android.hardware.camera2.CameraMetadata; 26 import android.hardware.camera2.CaptureRequest; 27 import android.media.AudioFormat; 28 import android.media.AudioRecord; 29 import android.media.Image; 30 import android.media.ImageReader; 31 import android.media.MediaRecorder; 32 import android.os.Handler; 33 import android.os.HandlerThread; 34 import android.os.PersistableBundle; 35 import android.os.SharedMemory; 36 import android.os.SystemClock; 37 import android.service.voice.VisualQueryDetectionService; 38 import android.util.Log; 39 import android.util.Range; 40 import android.util.Size; 41 import android.view.Surface; 42 43 import androidx.annotation.NonNull; 44 import androidx.annotation.Nullable; 45 46 import java.nio.ByteBuffer; 47 import java.util.Arrays; 48 import java.util.List; 49 import java.util.function.IntConsumer; 50 51 52 /** 53 * Sample VisualQueryDetectionService that captures camera frame and sends back partial query. 54 */ 55 public class SampleVisualQueryDetectionService extends VisualQueryDetectionService { 56 static final String TAG = "SVisualQueryDetectionSrvc"; 57 58 private final String FAKE_QUERY = "What is the weather today?"; 59 60 // Camera module related variables 61 // Set this to different values for different modes 62 private final int CAPTURE_MODE = CameraDevice.TEMPLATE_RECORD; 63 private String mCameraId; 64 private CaptureRequest.Builder mCaptureRequestBuilder; 65 private CameraCaptureSession mCameraCaptureSession; 66 private CameraDevice mCameraDevice; 67 private ImageReader mImageReader; 68 private Handler mCameraBackgroundHandler; 69 private HandlerThread mCameraBackgroundThread; 70 71 // audio module related variables 72 private static final int AUDIO_SAMPLE_RATE_IN_HZ = 16000; 73 private static final int AUDIO_CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO; 74 private static final int AUDIO_FORMAT = AudioFormat.ENCODING_DEFAULT; 75 private static final int BUFFER_SIZE = AUDIO_SAMPLE_RATE_IN_HZ; 76 private AudioRecord mAudioRecord; 77 private Handler mAudioBackgroundHandler; 78 private HandlerThread mAudioBackgroundThread; 79 80 81 @Override onStartDetection()82 public void onStartDetection() { 83 Log.i(TAG, "onStartDetection"); 84 startBackgroundThread(); 85 openCamera(); 86 mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.DEFAULT, 87 AUDIO_SAMPLE_RATE_IN_HZ, AUDIO_CHANNEL_CONFIG, AUDIO_FORMAT, BUFFER_SIZE); 88 } 89 90 @Override onStopDetection()91 public void onStopDetection() { 92 Log.i(TAG, "onStopDetection"); 93 releaseResources(); 94 stopBackgroundThread(); 95 } 96 97 @Override onUpdateState(@ullable PersistableBundle options, @Nullable SharedMemory sharedMemory, long callbackTimeoutMillis, @Nullable IntConsumer statusCallback)98 public void onUpdateState(@Nullable PersistableBundle options, 99 @Nullable SharedMemory sharedMemory, long callbackTimeoutMillis, 100 @Nullable IntConsumer statusCallback) { 101 Log.i(TAG, "onUpdateState"); 102 if (statusCallback != null) { 103 statusCallback.accept(0); 104 } 105 } 106 107 @Override onDestroy()108 public void onDestroy() { 109 Log.d(TAG, "Destroying visual query detection service"); 110 onStopDetection(); 111 } 112 113 /* Main logics of the system */ onReceiveImage(ImageReader reader)114 private void onReceiveImage(ImageReader reader){ 115 Log.i(TAG, "Image received."); 116 Image image = null; 117 try { 118 image = reader.acquireLatestImage(); 119 ByteBuffer buffer = image.getPlanes()[0].getBuffer(); 120 byte[] bytes = new byte[buffer.capacity()]; 121 buffer.get(bytes); 122 // Camera frame triggers attention 123 Log.i(TAG, "Image bytes received: " + Arrays.toString(bytes)); 124 gainedAttention(); 125 openMicrophone(); 126 } catch (Exception e) { 127 e.printStackTrace(); 128 } finally { 129 if (image != null) { 130 image.close(); 131 } 132 } 133 SystemClock.sleep(2_000); // wait 2 second to turn off attention 134 closeMicrophone(); 135 lostAttention(); 136 } 137 onReceiveAudio()138 private void onReceiveAudio(){ 139 try { 140 byte[] bytes = new byte[BUFFER_SIZE]; 141 int result = mAudioRecord.read(bytes, 0, BUFFER_SIZE); 142 if (result != AudioRecord.ERROR_INVALID_OPERATION) { 143 // The buffer can be all zeros due to initialization and reading delay 144 Log.i(TAG, "Audio bytes received: " + Arrays.toString(bytes)); 145 streamQuery(FAKE_QUERY); 146 finishQuery(); 147 } 148 } catch (Exception e) { 149 e.printStackTrace(); 150 } 151 SystemClock.sleep(2_000); //sleep so the buffer is a stable value 152 } 153 154 /* Sample Camera Module */ openCamera()155 private void openCamera() { 156 CameraManager manager = getSystemService(CameraManager.class); 157 Log.i(TAG, "Attempting to open camera"); 158 try { 159 mCameraId = manager.getCameraIdList()[0]; //get front facing camera 160 CameraCharacteristics characteristics = manager.getCameraCharacteristics(mCameraId); 161 Size imageSize = characteristics.get( 162 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) 163 .getOutputSizes(ImageFormat.JPEG)[0]; 164 initializeImageReader(imageSize.getWidth(), imageSize.getHeight()); 165 manager.openCamera(mCameraId, stateCallback, mCameraBackgroundHandler); 166 } catch (CameraAccessException e) { 167 e.printStackTrace(); 168 } 169 Log.i(TAG, "Camera opened."); 170 } 171 initializeImageReader(int width, int height)172 private void initializeImageReader(int width, int height) { 173 // Initialize image reader 174 mImageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 2); 175 ImageReader.OnImageAvailableListener readerListener = 176 new ImageReader.OnImageAvailableListener() { 177 @Override 178 public void onImageAvailable(ImageReader reader) { 179 onReceiveImage(reader); 180 } 181 } ; 182 mImageReader.setOnImageAvailableListener(readerListener, mCameraBackgroundHandler); 183 } 184 185 private final CameraDevice.StateCallback stateCallback = new CameraDevice.StateCallback() { 186 @Override 187 public void onOpened(CameraDevice camera) { 188 // This is called when the camera is open 189 Log.i(TAG, "onCameraOpened"); 190 mCameraDevice = camera; 191 createCameraPreview(); 192 } 193 194 @Override 195 public void onDisconnected(CameraDevice camera) { 196 mCameraDevice.close(); 197 } 198 199 @Override 200 public void onError(CameraDevice camera, int error) { 201 mCameraDevice.close(); 202 mCameraDevice = null; 203 } 204 }; 205 createCameraPreview()206 private void createCameraPreview() { 207 Range<Integer> fpsRange = new Range<>(1,2); 208 try { 209 mCaptureRequestBuilder = mCameraDevice.createCaptureRequest(CAPTURE_MODE); 210 Surface imageSurface = mImageReader.getSurface(); 211 mCaptureRequestBuilder.addTarget(imageSurface); 212 mCaptureRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 213 mCameraDevice.createCaptureSession(List.of(imageSurface), 214 new CameraCaptureSession.StateCallback() { 215 @Override 216 public void onConfigured( 217 @NonNull CameraCaptureSession cameraCaptureSession) { 218 //The camera is already closed 219 if (mCameraDevice == null) { 220 return; 221 } 222 // When the session is ready, we start displaying the preview. 223 mCameraCaptureSession = cameraCaptureSession; 224 updatePreview(); 225 Log.i(TAG, "Capture session configured."); 226 } 227 228 @Override 229 public void onConfigureFailed( 230 @NonNull CameraCaptureSession cameraCaptureSession) { 231 //No-op 232 } 233 }, null); 234 } catch (CameraAccessException e) { 235 e.printStackTrace(); 236 } 237 Log.i(TAG, "Camera preview created."); 238 } 239 updatePreview()240 private void updatePreview() { 241 if (null == mCameraDevice) { 242 Log.e(TAG, "updatePreview error, return"); 243 } 244 mCaptureRequestBuilder.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO); 245 try { 246 if (CAPTURE_MODE == CameraDevice.TEMPLATE_STILL_CAPTURE 247 || CAPTURE_MODE == CameraDevice.TEMPLATE_VIDEO_SNAPSHOT) { 248 mCameraCaptureSession.capture(mCaptureRequestBuilder.build(), null, 249 mCameraBackgroundHandler); 250 } else if (CAPTURE_MODE == CameraDevice.TEMPLATE_RECORD 251 || CAPTURE_MODE == CameraDevice.TEMPLATE_PREVIEW){ 252 mCameraCaptureSession.setRepeatingRequest(mCaptureRequestBuilder.build(), null, 253 mCameraBackgroundHandler); 254 } else { 255 throw new IllegalStateException("Capture mode not supported."); 256 } 257 } catch (Exception e) { 258 e.printStackTrace(); 259 } 260 } 261 262 /* Sample Microphone Module */ openMicrophone()263 private void openMicrophone() { 264 mAudioRecord.startRecording(); 265 mAudioBackgroundHandler.post(this::onReceiveAudio); 266 } 267 closeMicrophone()268 private void closeMicrophone() { 269 if (mAudioRecord != null) { 270 mAudioRecord.stop(); 271 } 272 } 273 releaseResources()274 private void releaseResources() { 275 mCameraId = null; 276 mCaptureRequestBuilder = null; 277 // Release mCameraCaptureSession 278 mCameraCaptureSession.close(); 279 mCameraCaptureSession = null; 280 // Release mCameraDevice 281 mCameraDevice.close(); 282 mCameraDevice = null; 283 // Release mImageReader 284 mImageReader.close(); 285 mImageReader = null; 286 // Release mAudioRecord 287 mAudioRecord.release(); 288 mAudioRecord = null; 289 } 290 // Handlers startBackgroundThread()291 private void startBackgroundThread() { 292 mCameraBackgroundThread = new HandlerThread("Camera Background Thread"); 293 mCameraBackgroundThread.start(); 294 mCameraBackgroundHandler = new Handler(mCameraBackgroundThread.getLooper()); 295 mAudioBackgroundThread = new HandlerThread("Audio Background Thread"); 296 mAudioBackgroundThread.start(); 297 mAudioBackgroundHandler = new Handler(mAudioBackgroundThread.getLooper()); 298 } 299 stopBackgroundThread()300 private void stopBackgroundThread() { 301 mCameraBackgroundThread.quitSafely(); 302 try { 303 mCameraBackgroundThread.join(); 304 mCameraBackgroundThread = null; 305 mCameraBackgroundHandler = null; 306 } catch (InterruptedException e) { 307 Log.e(TAG, "Failed to stop camera thread."); 308 } 309 mAudioBackgroundThread.quitSafely(); 310 try { 311 mAudioBackgroundThread.join(); 312 mAudioBackgroundThread = null; 313 mAudioBackgroundHandler = null; 314 } catch (InterruptedException e) { 315 Log.e(TAG, "Failed to stop audio thread."); 316 } 317 } 318 }