1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.example.android.vdmdemo.client; 18 19 import android.content.Context; 20 import android.hardware.camera2.CameraAccessException; 21 import android.hardware.camera2.CameraCaptureSession; 22 import android.hardware.camera2.CameraCharacteristics; 23 import android.hardware.camera2.CameraDevice; 24 import android.hardware.camera2.CameraManager; 25 import android.hardware.camera2.CaptureRequest; 26 import android.hardware.camera2.TotalCaptureResult; 27 import android.hardware.camera2.params.OutputConfiguration; 28 import android.hardware.camera2.params.SessionConfiguration; 29 import android.util.ArrayMap; 30 import android.util.Log; 31 import android.view.Surface; 32 33 import androidx.annotation.GuardedBy; 34 import androidx.annotation.NonNull; 35 36 import com.example.android.vdmdemo.common.RemoteEventProto; 37 import com.example.android.vdmdemo.common.RemoteIo; 38 import com.example.android.vdmdemo.common.VideoManager; 39 40 import dagger.hilt.android.qualifiers.ApplicationContext; 41 import dagger.hilt.android.scopes.ActivityScoped; 42 43 import java.util.ArrayList; 44 import java.util.List; 45 import java.util.Objects; 46 import java.util.Optional; 47 import java.util.concurrent.Executor; 48 import java.util.concurrent.Executors; 49 import java.util.function.Consumer; 50 51 import javax.inject.Inject; 52 53 @ActivityScoped 54 final class VirtualCameraController implements AutoCloseable { 55 private static final String TAG = VirtualCameraController.class.getSimpleName(); 56 57 private static final String NO_CAMERA_PERMISSION_LOG_MSG = 58 "VDM Client app doesn't have access to camera"; 59 60 private static final int CAMERA_WIDTH = 640; 61 private static final int CAMERA_HEIGHT = 480; 62 private static final int FPS = 30; 63 64 private final RemoteIo mRemoteIo; 65 private final Consumer<RemoteEventProto.RemoteEvent> mRemoteEventConsumer = 66 this::processRemoteEvent; 67 private final CameraManager mCameraManager; 68 69 private final ArrayList<String> mExposedCameras = new ArrayList<>(2); 70 71 private final Object mLock = new Object(); 72 73 @GuardedBy("mLock") 74 private final ArrayMap<String, CameraStreamer> mCameraStreamerMap = new ArrayMap<>(); 75 76 private final Executor mExecutor = Executors.newSingleThreadExecutor(); 77 78 @Inject VirtualCameraController(@pplicationContext Context context, RemoteIo remoteIo)79 VirtualCameraController(@ApplicationContext Context context, RemoteIo remoteIo) { 80 mRemoteIo = remoteIo; 81 mRemoteIo.addMessageConsumer(mRemoteEventConsumer); 82 mCameraManager = Objects.requireNonNull(context.getSystemService(CameraManager.class)); 83 84 try { 85 getFrontCamera().ifPresent(mExposedCameras::add); 86 getBackCamera().ifPresent(mExposedCameras::add); 87 } catch (CameraAccessException exception) { 88 Log.e(TAG, NO_CAMERA_PERMISSION_LOG_MSG, exception); 89 } 90 } 91 getCameraCapabilities()92 List<RemoteEventProto.CameraCapabilities> getCameraCapabilities() { 93 ArrayList<RemoteEventProto.CameraCapabilities> cameraCapabilities = new ArrayList<>( 94 mExposedCameras.size()); 95 try { 96 for (String cameraId : mExposedCameras) { 97 cameraCapabilities.add(getCapabilitiesForCamera(cameraId)); 98 } 99 } catch (CameraAccessException exception) { 100 Log.e(TAG, "VDM Client doesn't have camera permission", exception); 101 } 102 Log.d(TAG, "Camera capabilities" + cameraCapabilities); 103 return cameraCapabilities; 104 } 105 getFrontCamera()106 Optional<String> getFrontCamera() throws CameraAccessException { 107 return getCameraWithLensFacing(CameraCharacteristics.LENS_FACING_FRONT); 108 } 109 getBackCamera()110 Optional<String> getBackCamera() throws CameraAccessException { 111 return getCameraWithLensFacing(CameraCharacteristics.LENS_FACING_BACK); 112 } 113 getCameraWithLensFacing(int lensFacing)114 Optional<String> getCameraWithLensFacing(int lensFacing) throws CameraAccessException { 115 for (String cameraId : mCameraManager.getCameraIdList()) { 116 CameraCharacteristics characteristics = mCameraManager.getCameraCharacteristics( 117 cameraId); 118 119 if (Objects.equals(characteristics.get(CameraCharacteristics.LENS_FACING), 120 lensFacing)) { 121 return Optional.of(cameraId); 122 } 123 } 124 return Optional.empty(); 125 } 126 getCapabilitiesForCamera(String cameraId)127 RemoteEventProto.CameraCapabilities getCapabilitiesForCamera(String cameraId) 128 throws CameraAccessException { 129 CameraCharacteristics characteristics = mCameraManager.getCameraCharacteristics(cameraId); 130 131 132 return RemoteEventProto.CameraCapabilities.newBuilder() 133 .setCameraId(cameraId) 134 .setWidth(CAMERA_WIDTH) 135 .setHeight(CAMERA_HEIGHT) 136 .setFps(FPS) 137 .setLensFacing(Objects.requireNonNull( 138 characteristics.get( 139 CameraCharacteristics.LENS_FACING))) 140 .setSensorOrientation(Objects.requireNonNull( 141 characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION))) 142 .build(); 143 } 144 processRemoteEvent(RemoteEventProto.RemoteEvent remoteEvent)145 private void processRemoteEvent(RemoteEventProto.RemoteEvent remoteEvent) { 146 try { 147 if (remoteEvent.hasStartCameraStream()) { 148 startCameraStream(remoteEvent.getStartCameraStream().getCameraId()); 149 } 150 if (remoteEvent.hasStopCameraStream()) { 151 stopCameraStream(remoteEvent.getStopCameraStream().getCameraId()); 152 } 153 } catch (CameraAccessException exception) { 154 Log.e(TAG, NO_CAMERA_PERMISSION_LOG_MSG, exception); 155 } 156 } 157 startCameraStream(String cameraId)158 private void startCameraStream(String cameraId) throws CameraAccessException { 159 Log.d(TAG, "Start camera stream " + cameraId); 160 synchronized (mLock) { 161 if (mCameraStreamerMap.containsKey(cameraId)) { 162 return; 163 } 164 CameraStreamer cameraStreamer = new CameraStreamer(cameraId); 165 mCameraStreamerMap.put(cameraId, cameraStreamer); 166 cameraStreamer.start(); 167 } 168 } 169 stopCameraStream(String cameraId)170 private void stopCameraStream(String cameraId) { 171 Log.d(TAG, "Stop camera stream " + cameraId); 172 synchronized (mLock) { 173 if (!mCameraStreamerMap.containsKey(cameraId)) { 174 return; 175 } 176 177 CameraStreamer streamer = Objects.requireNonNull(mCameraStreamerMap.get(cameraId)); 178 streamer.stop(); 179 mCameraStreamerMap.remove(cameraId); 180 } 181 } 182 183 @Override close()184 public void close() throws Exception { 185 mRemoteIo.removeMessageConsumer(mRemoteEventConsumer); 186 synchronized (mLock) { 187 mCameraStreamerMap.values().forEach(CameraStreamer::stop); 188 mCameraStreamerMap.clear(); 189 } 190 } 191 192 private class CameraStreamer { 193 private final String mCameraId; 194 private final VideoManager mVideoManager; 195 196 private final Object mLock = new Object(); 197 198 @GuardedBy("mLock") 199 private CameraDevice mCameraDevice; 200 201 @GuardedBy("mLock") 202 private Surface mSurface; 203 204 @GuardedBy("mLock") 205 private CameraCaptureSession mCameraCaptureSession; 206 207 private final CameraDevice.StateCallback mDeviceStateCallback = 208 new CameraDevice.StateCallback() { 209 @Override 210 public void onOpened(@NonNull CameraDevice camera) { 211 synchronized (mLock) { 212 mCameraDevice = camera; 213 mSurface = mVideoManager.createInputSurface(CAMERA_WIDTH, CAMERA_HEIGHT, 214 FPS); 215 mVideoManager.startEncoding(); 216 217 OutputConfiguration outputConfiguration = new OutputConfiguration( 218 mSurface); 219 SessionConfiguration sessionConfig = new SessionConfiguration( 220 SessionConfiguration.SESSION_REGULAR, 221 List.of(outputConfiguration), mExecutor, mSessionStateCallback); 222 try { 223 camera.createCaptureSession(sessionConfig); 224 } catch (CameraAccessException exception) { 225 Log.e(TAG, NO_CAMERA_PERMISSION_LOG_MSG, exception); 226 } 227 } 228 } 229 230 @Override 231 public void onDisconnected(@NonNull CameraDevice camera) { 232 Log.d(TAG, "Camera device " + mCameraId + " disconnected"); 233 } 234 235 @Override 236 public void onError(@NonNull CameraDevice camera, int error) { 237 Log.e(TAG, "Camera device " + mCameraId + " error: " + error); 238 } 239 }; 240 241 private final CameraCaptureSession.StateCallback mSessionStateCallback = 242 new CameraCaptureSession.StateCallback() { 243 @Override 244 public void onConfigured(@NonNull CameraCaptureSession session) { 245 synchronized (mLock) { 246 try { 247 mCameraCaptureSession = session; 248 CaptureRequest.Builder builder = mCameraDevice.createCaptureRequest( 249 CameraDevice.TEMPLATE_PREVIEW); 250 builder.addTarget(mSurface); 251 252 session.setSingleRepeatingRequest(builder.build(), mExecutor, 253 mCaptureCallback); 254 } catch (CameraAccessException exception) { 255 Log.e(TAG, NO_CAMERA_PERMISSION_LOG_MSG, exception); 256 } 257 } 258 } 259 260 @Override 261 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 262 Log.e(TAG, "Configuration failed!"); 263 } 264 }; 265 266 private final CameraCaptureSession.CaptureCallback mCaptureCallback = 267 new CameraCaptureSession.CaptureCallback() { 268 @Override 269 public void onCaptureCompleted( 270 @NonNull CameraCaptureSession session, 271 @NonNull CaptureRequest request, 272 @NonNull TotalCaptureResult result) { 273 super.onCaptureCompleted(session, request, result); 274 Log.d(TAG, "onCaptureCompleted"); 275 } 276 }; 277 CameraStreamer(String cameraId)278 CameraStreamer(String cameraId) { 279 mCameraId = cameraId; 280 mVideoManager = VideoManager.createCameraEncoder(cameraId, mRemoteIo, false); 281 } 282 start()283 void start() throws CameraAccessException { 284 mCameraManager.openCamera(mCameraId, mExecutor, mDeviceStateCallback); 285 } 286 stop()287 void stop() { 288 synchronized (mLock) { 289 try { 290 mVideoManager.stop(); 291 292 if (mCameraCaptureSession != null) { 293 mCameraCaptureSession.stopRepeating(); 294 mCameraCaptureSession.close(); 295 } 296 297 if (mCameraDevice != null) { 298 mCameraDevice.close(); 299 } 300 } catch (CameraAccessException exception) { 301 Log.e(TAG, NO_CAMERA_PERMISSION_LOG_MSG, exception); 302 } 303 } 304 } 305 } 306 } 307