1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.RequiresPermission; 22 import android.content.Context; 23 import android.graphics.ImageFormat; 24 import android.graphics.SurfaceTexture; 25 import android.hardware.HardwareBuffer; 26 import android.hardware.SyncFence; 27 import android.hardware.camera2.CameraAccessException; 28 import android.hardware.camera2.CameraCaptureSession; 29 import android.hardware.camera2.CameraCharacteristics; 30 import android.hardware.camera2.CameraDevice; 31 import android.hardware.camera2.CameraExtensionCharacteristics; 32 import android.hardware.camera2.CameraExtensionSession; 33 import android.hardware.camera2.CameraManager; 34 import android.hardware.camera2.CaptureFailure; 35 import android.hardware.camera2.CaptureRequest; 36 import android.hardware.camera2.CaptureResult; 37 import android.hardware.camera2.TotalCaptureResult; 38 import android.hardware.camera2.extension.CaptureBundle; 39 import android.hardware.camera2.extension.CaptureStageImpl; 40 import android.hardware.camera2.extension.ICaptureProcessorImpl; 41 import android.hardware.camera2.extension.IImageCaptureExtenderImpl; 42 import android.hardware.camera2.extension.IInitializeSessionCallback; 43 import android.hardware.camera2.extension.IPreviewExtenderImpl; 44 import android.hardware.camera2.extension.IProcessResultImpl; 45 import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; 46 import android.hardware.camera2.extension.ParcelImage; 47 import android.hardware.camera2.params.DynamicRangeProfiles; 48 import android.hardware.camera2.params.ExtensionSessionConfiguration; 49 import android.hardware.camera2.params.OutputConfiguration; 50 import android.hardware.camera2.params.SessionConfiguration; 51 import android.hardware.camera2.utils.SurfaceUtils; 52 import android.media.Image; 53 import android.media.ImageReader; 54 import android.media.ImageWriter; 55 import android.os.Binder; 56 import android.os.Handler; 57 import android.os.HandlerThread; 58 import android.os.RemoteException; 59 import android.util.Log; 60 import android.util.LongSparseArray; 61 import android.util.Pair; 62 import android.util.Size; 63 import android.view.Surface; 64 65 import java.io.Closeable; 66 import java.io.IOException; 67 import java.util.ArrayList; 68 import java.util.HashMap; 69 import java.util.List; 70 import java.util.Map; 71 import java.util.Set; 72 import java.util.concurrent.Executor; 73 74 public final class CameraExtensionSessionImpl extends CameraExtensionSession { 75 private static final int PREVIEW_QUEUE_SIZE = 10; 76 private static final String TAG = "CameraExtensionSessionImpl"; 77 78 private final Executor mExecutor; 79 private final CameraDevice mCameraDevice; 80 private final long mExtensionClientId; 81 private final IImageCaptureExtenderImpl mImageExtender; 82 private final IPreviewExtenderImpl mPreviewExtender; 83 private final Handler mHandler; 84 private final HandlerThread mHandlerThread; 85 private final StateCallback mCallbacks; 86 private final List<Size> mSupportedPreviewSizes; 87 private final InitializeSessionHandler mInitializeHandler; 88 private final int mSessionId; 89 private final Set<CaptureRequest.Key> mSupportedRequestKeys; 90 private final Set<CaptureResult.Key> mSupportedResultKeys; 91 private boolean mCaptureResultsSupported; 92 93 private CameraCaptureSession mCaptureSession = null; 94 private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface; 95 private Surface mCameraBurstSurface, mClientCaptureSurface; 96 private ImageReader mRepeatingRequestImageReader = null; 97 private ImageReader mBurstCaptureImageReader = null; 98 private ImageReader mStubCaptureImageReader = null; 99 private ImageWriter mRepeatingRequestImageWriter = null; 100 private CameraOutputImageCallback mRepeatingRequestImageCallback = null; 101 private CameraOutputImageCallback mBurstCaptureImageCallback = null; 102 103 private CameraExtensionJpegProcessor mImageJpegProcessor = null; 104 private ICaptureProcessorImpl mImageProcessor = null; 105 private CameraExtensionForwardProcessor mPreviewImageProcessor = null; 106 private IRequestUpdateProcessorImpl mPreviewRequestUpdateProcessor = null; 107 private int mPreviewProcessorType = IPreviewExtenderImpl.PROCESSOR_TYPE_NONE; 108 109 private boolean mInitialized; 110 // Enable/Disable internal preview/(repeating request). Extensions expect 111 // that preview/(repeating request) is enabled and active at any point in time. 112 // In case the client doesn't explicitly enable repeating requests, the framework 113 // will do so internally. 114 private boolean mInternalRepeatingRequestEnabled = true; 115 116 // Lock to synchronize cross-thread access to device public interface 117 final Object mInterfaceLock = new Object(); // access from this class and Session only! 118 nativeGetSurfaceFormat(Surface surface)119 private static int nativeGetSurfaceFormat(Surface surface) { 120 return SurfaceUtils.getSurfaceFormat(surface); 121 } 122 123 /** 124 * @hide 125 */ 126 @RequiresPermission(android.Manifest.permission.CAMERA) createCameraExtensionSession( @onNull CameraDevice cameraDevice, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId)127 public static CameraExtensionSessionImpl createCameraExtensionSession( 128 @NonNull CameraDevice cameraDevice, 129 @NonNull Context ctx, 130 @NonNull ExtensionSessionConfiguration config, 131 int sessionId) 132 throws CameraAccessException, RemoteException { 133 long clientId = CameraExtensionCharacteristics.registerClient(ctx); 134 if (clientId < 0) { 135 throw new UnsupportedOperationException("Unsupported extension!"); 136 } 137 138 String cameraId = cameraDevice.getId(); 139 CameraManager manager = ctx.getSystemService(CameraManager.class); 140 CameraCharacteristics chars = manager.getCameraCharacteristics(cameraId); 141 CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, 142 cameraId, chars); 143 144 if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), 145 config.getExtension(), chars)) { 146 throw new UnsupportedOperationException("Unsupported extension type: " + 147 config.getExtension()); 148 } 149 150 if (config.getOutputConfigurations().isEmpty() || 151 config.getOutputConfigurations().size() > 2) { 152 throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + 153 config.getOutputConfigurations().size() + " expected <= 2"); 154 } 155 156 for (OutputConfiguration c : config.getOutputConfigurations()) { 157 if (c.getDynamicRangeProfile() != DynamicRangeProfiles.STANDARD) { 158 throw new IllegalArgumentException("Unsupported dynamic range profile: " + 159 c.getDynamicRangeProfile()); 160 } 161 if (c.getStreamUseCase() != 162 CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT) { 163 throw new IllegalArgumentException("Unsupported stream use case: " + 164 c.getStreamUseCase()); 165 } 166 } 167 168 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 169 CameraExtensionCharacteristics.initializeExtension(config.getExtension()); 170 171 int suitableSurfaceCount = 0; 172 List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( 173 config.getExtension(), SurfaceTexture.class); 174 Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface( 175 config.getOutputConfigurations(), supportedPreviewSizes); 176 if (repeatingRequestSurface != null) { 177 suitableSurfaceCount++; 178 } 179 180 HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>(); 181 for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 182 List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes( 183 config.getExtension(), format); 184 if (supportedSizes != null) { 185 supportedCaptureSizes.put(format, supportedSizes); 186 } 187 } 188 Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface( 189 config.getOutputConfigurations(), supportedCaptureSizes); 190 if (burstCaptureSurface != null) { 191 suitableSurfaceCount++; 192 } 193 194 if (suitableSurfaceCount != config.getOutputConfigurations().size()) { 195 throw new IllegalArgumentException("One or more unsupported output surfaces found!"); 196 } 197 198 extenders.first.init(cameraId, chars.getNativeMetadata()); 199 extenders.first.onInit(cameraId, chars.getNativeMetadata()); 200 extenders.second.init(cameraId, chars.getNativeMetadata()); 201 extenders.second.onInit(cameraId, chars.getNativeMetadata()); 202 203 CameraExtensionSessionImpl session = new CameraExtensionSessionImpl( 204 extenders.second, 205 extenders.first, 206 supportedPreviewSizes, 207 clientId, 208 cameraDevice, 209 repeatingRequestSurface, 210 burstCaptureSurface, 211 config.getStateCallback(), 212 config.getExecutor(), 213 sessionId, 214 extensionChars.getAvailableCaptureRequestKeys(config.getExtension()), 215 extensionChars.getAvailableCaptureResultKeys(config.getExtension())); 216 217 session.initialize(); 218 219 return session; 220 } 221 CameraExtensionSessionImpl(@onNull IImageCaptureExtenderImpl imageExtender, @NonNull IPreviewExtenderImpl previewExtender, @NonNull List<Size> previewSizes, long extensionClientId, @NonNull CameraDevice cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @NonNull StateCallback callback, @NonNull Executor executor, int sessionId, @NonNull Set<CaptureRequest.Key> requestKeys, @Nullable Set<CaptureResult.Key> resultKeys)222 public CameraExtensionSessionImpl(@NonNull IImageCaptureExtenderImpl imageExtender, 223 @NonNull IPreviewExtenderImpl previewExtender, 224 @NonNull List<Size> previewSizes, 225 long extensionClientId, 226 @NonNull CameraDevice cameraDevice, 227 @Nullable Surface repeatingRequestSurface, 228 @Nullable Surface burstCaptureSurface, 229 @NonNull StateCallback callback, 230 @NonNull Executor executor, 231 int sessionId, 232 @NonNull Set<CaptureRequest.Key> requestKeys, 233 @Nullable Set<CaptureResult.Key> resultKeys) { 234 mExtensionClientId = extensionClientId; 235 mImageExtender = imageExtender; 236 mPreviewExtender = previewExtender; 237 mCameraDevice = cameraDevice; 238 mCallbacks = callback; 239 mExecutor = executor; 240 mClientRepeatingRequestSurface = repeatingRequestSurface; 241 mClientCaptureSurface = burstCaptureSurface; 242 mSupportedPreviewSizes = previewSizes; 243 mHandlerThread = new HandlerThread(TAG); 244 mHandlerThread.start(); 245 mHandler = new Handler(mHandlerThread.getLooper()); 246 mInitialized = false; 247 mInitializeHandler = new InitializeSessionHandler(); 248 mSessionId = sessionId; 249 mSupportedRequestKeys = requestKeys; 250 mSupportedResultKeys = resultKeys; 251 mCaptureResultsSupported = !resultKeys.isEmpty(); 252 } 253 initializeRepeatingRequestPipeline()254 private void initializeRepeatingRequestPipeline() throws RemoteException { 255 CameraExtensionUtils.SurfaceInfo repeatingSurfaceInfo = 256 new CameraExtensionUtils.SurfaceInfo(); 257 mPreviewProcessorType = mPreviewExtender.getProcessorType(); 258 if (mClientRepeatingRequestSurface != null) { 259 repeatingSurfaceInfo = CameraExtensionUtils.querySurface( 260 mClientRepeatingRequestSurface); 261 } else { 262 // Make the intermediate surface behave as any regular 'SurfaceTexture' 263 CameraExtensionUtils.SurfaceInfo captureSurfaceInfo = CameraExtensionUtils.querySurface( 264 mClientCaptureSurface); 265 Size captureSize = new Size(captureSurfaceInfo.mWidth, captureSurfaceInfo.mHeight); 266 Size previewSize = findSmallestAspectMatchedSize(mSupportedPreviewSizes, captureSize); 267 repeatingSurfaceInfo.mWidth = previewSize.getWidth(); 268 repeatingSurfaceInfo.mHeight = previewSize.getHeight(); 269 repeatingSurfaceInfo.mUsage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE; 270 } 271 272 if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { 273 try { 274 mPreviewImageProcessor = new CameraExtensionForwardProcessor( 275 mPreviewExtender.getPreviewImageProcessor(), repeatingSurfaceInfo.mFormat, 276 repeatingSurfaceInfo.mUsage, mHandler); 277 } catch (ClassCastException e) { 278 throw new UnsupportedOperationException("Failed casting preview processor!"); 279 } 280 mPreviewImageProcessor.onImageFormatUpdate( 281 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); 282 mPreviewImageProcessor.onResolutionUpdate(new Size(repeatingSurfaceInfo.mWidth, 283 repeatingSurfaceInfo.mHeight)); 284 mPreviewImageProcessor.onOutputSurface(null, -1); 285 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 286 repeatingSurfaceInfo.mHeight, 287 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, PREVIEW_QUEUE_SIZE, 288 repeatingSurfaceInfo.mUsage); 289 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 290 } else if (mPreviewProcessorType == 291 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { 292 try { 293 mPreviewRequestUpdateProcessor = mPreviewExtender.getRequestUpdateProcessor(); 294 } catch (ClassCastException e) { 295 throw new UnsupportedOperationException("Failed casting preview processor!"); 296 } 297 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 298 repeatingSurfaceInfo.mHeight, 299 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, 300 PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); 301 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 302 android.hardware.camera2.extension.Size sz = 303 new android.hardware.camera2.extension.Size(); 304 sz.width = repeatingSurfaceInfo.mWidth; 305 sz.height = repeatingSurfaceInfo.mHeight; 306 mPreviewRequestUpdateProcessor.onResolutionUpdate(sz); 307 mPreviewRequestUpdateProcessor.onImageFormatUpdate( 308 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 309 } else { 310 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 311 repeatingSurfaceInfo.mHeight, 312 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, 313 PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); 314 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 315 } 316 mRepeatingRequestImageCallback = new CameraOutputImageCallback( 317 mRepeatingRequestImageReader); 318 mRepeatingRequestImageReader 319 .setOnImageAvailableListener(mRepeatingRequestImageCallback, mHandler); 320 } 321 initializeBurstCapturePipeline()322 private void initializeBurstCapturePipeline() throws RemoteException { 323 mImageProcessor = mImageExtender.getCaptureProcessor(); 324 if ((mImageProcessor == null) && (mImageExtender.getMaxCaptureStage() != 1)) { 325 throw new UnsupportedOperationException("Multiple stages expected without" + 326 " a valid capture processor!"); 327 } 328 329 if (mImageProcessor != null) { 330 if (mClientCaptureSurface != null) { 331 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface( 332 mClientCaptureSurface); 333 if (surfaceInfo.mFormat == ImageFormat.JPEG) { 334 mImageJpegProcessor = new CameraExtensionJpegProcessor(mImageProcessor); 335 mImageProcessor = mImageJpegProcessor; 336 } 337 mBurstCaptureImageReader = ImageReader.newInstance(surfaceInfo.mWidth, 338 surfaceInfo.mHeight, CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 339 mImageExtender.getMaxCaptureStage()); 340 } else { 341 // The client doesn't intend to trigger multi-frame capture, however the 342 // image extender still needs to get initialized and the camera still capture 343 // stream configured for the repeating request processing to work. 344 mBurstCaptureImageReader = ImageReader.newInstance( 345 mRepeatingRequestImageReader.getWidth(), 346 mRepeatingRequestImageReader.getHeight(), 347 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); 348 // The still capture output is not going to be used but we still need a valid 349 // surface to register. 350 mStubCaptureImageReader = ImageReader.newInstance( 351 mRepeatingRequestImageReader.getWidth(), 352 mRepeatingRequestImageReader.getHeight(), 353 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); 354 mImageProcessor.onOutputSurface(mStubCaptureImageReader.getSurface(), 355 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); 356 } 357 358 mBurstCaptureImageCallback = new CameraOutputImageCallback(mBurstCaptureImageReader); 359 mBurstCaptureImageReader.setOnImageAvailableListener(mBurstCaptureImageCallback, 360 mHandler); 361 mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); 362 android.hardware.camera2.extension.Size sz = 363 new android.hardware.camera2.extension.Size(); 364 sz.width = mBurstCaptureImageReader.getWidth(); 365 sz.height = mBurstCaptureImageReader.getHeight(); 366 mImageProcessor.onResolutionUpdate(sz); 367 mImageProcessor.onImageFormatUpdate(mBurstCaptureImageReader.getImageFormat()); 368 } else { 369 if (mClientCaptureSurface != null) { 370 // Redirect camera output directly in to client output surface 371 mCameraBurstSurface = mClientCaptureSurface; 372 } else { 373 // The client doesn't intend to trigger multi-frame capture, however the 374 // image extender still needs to get initialized and the camera still capture 375 // stream configured for the repeating request processing to work. 376 mBurstCaptureImageReader = ImageReader.newInstance( 377 mRepeatingRequestImageReader.getWidth(), 378 mRepeatingRequestImageReader.getHeight(), 379 // Camera devices accept only Jpeg output if the image processor is null 380 ImageFormat.JPEG, 1); 381 mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); 382 } 383 } 384 } 385 finishPipelineInitialization()386 private void finishPipelineInitialization() throws RemoteException { 387 if (mClientRepeatingRequestSurface != null) { 388 if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { 389 mPreviewRequestUpdateProcessor.onOutputSurface(mClientRepeatingRequestSurface, 390 nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); 391 mRepeatingRequestImageWriter = ImageWriter.newInstance( 392 mClientRepeatingRequestSurface, 393 PREVIEW_QUEUE_SIZE, 394 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 395 } else if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_NONE) { 396 mRepeatingRequestImageWriter = ImageWriter.newInstance( 397 mClientRepeatingRequestSurface, 398 PREVIEW_QUEUE_SIZE, 399 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 400 } 401 } 402 if ((mImageProcessor != null) && (mClientCaptureSurface != null)) { 403 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface( 404 mClientCaptureSurface); 405 mImageProcessor.onOutputSurface(mClientCaptureSurface, surfaceInfo.mFormat); 406 } 407 } 408 409 /** 410 * @hide 411 */ initialize()412 public synchronized void initialize() throws CameraAccessException, RemoteException { 413 if (mInitialized) { 414 Log.d(TAG, 415 "Session already initialized"); 416 return; 417 } 418 419 ArrayList<CaptureStageImpl> sessionParamsList = new ArrayList<>(); 420 ArrayList<OutputConfiguration> outputList = new ArrayList<>(); 421 initializeRepeatingRequestPipeline(); 422 outputList.add(new OutputConfiguration(mCameraRepeatingSurface)); 423 CaptureStageImpl previewSessionParams = mPreviewExtender.onPresetSession(); 424 if (previewSessionParams != null) { 425 sessionParamsList.add(previewSessionParams); 426 } 427 initializeBurstCapturePipeline(); 428 outputList.add(new OutputConfiguration(mCameraBurstSurface)); 429 CaptureStageImpl stillCaptureSessionParams = mImageExtender.onPresetSession(); 430 if (stillCaptureSessionParams != null) { 431 sessionParamsList.add(stillCaptureSessionParams); 432 } 433 434 SessionConfiguration sessionConfig = new SessionConfiguration( 435 SessionConfiguration.SESSION_REGULAR, 436 outputList, 437 new CameraExtensionUtils.HandlerExecutor(mHandler), 438 new SessionStateHandler()); 439 440 if (!sessionParamsList.isEmpty()) { 441 CaptureRequest sessionParamRequest = createRequest(mCameraDevice, sessionParamsList, 442 null, CameraDevice.TEMPLATE_PREVIEW); 443 sessionConfig.setSessionParameters(sessionParamRequest); 444 } 445 446 mCameraDevice.createCaptureSession(sessionConfig); 447 } 448 449 @Override getDevice()450 public @NonNull CameraDevice getDevice() { 451 synchronized (mInterfaceLock) { 452 return mCameraDevice; 453 } 454 } 455 456 @Override setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)457 public int setRepeatingRequest(@NonNull CaptureRequest request, 458 @NonNull Executor executor, 459 @NonNull ExtensionCaptureCallback listener) 460 throws CameraAccessException { 461 synchronized (mInterfaceLock) { 462 if (!mInitialized) { 463 throw new IllegalStateException("Uninitialized component"); 464 } 465 466 if (mClientRepeatingRequestSurface == null) { 467 throw new IllegalArgumentException("No registered preview surface"); 468 } 469 470 if (!request.containsTarget(mClientRepeatingRequestSurface) || 471 (request.getTargets().size() != 1)) { 472 throw new IllegalArgumentException("Invalid repeating request output target!"); 473 } 474 475 mInternalRepeatingRequestEnabled = false; 476 try { 477 return setRepeatingRequest(mPreviewExtender.getCaptureStage(), 478 new PreviewRequestHandler(request, executor, listener, 479 mRepeatingRequestImageCallback), request); 480 } catch (RemoteException e) { 481 Log.e(TAG, "Failed to set repeating request! Extension service does not " 482 + "respond"); 483 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 484 } 485 } 486 } 487 compileInitialRequestList()488 private ArrayList<CaptureStageImpl> compileInitialRequestList() { 489 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 490 try { 491 CaptureStageImpl initialPreviewParams = mPreviewExtender.onEnableSession(); 492 if (initialPreviewParams != null) { 493 captureStageList.add(initialPreviewParams); 494 } 495 496 CaptureStageImpl initialStillCaptureParams = mImageExtender.onEnableSession(); 497 if (initialStillCaptureParams != null) { 498 captureStageList.add(initialStillCaptureParams); 499 } 500 } catch (RemoteException e) { 501 Log.e(TAG, "Failed to initialize session parameters! Extension service does not" 502 + " respond!"); 503 } 504 505 return captureStageList; 506 } 507 createBurstRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap)508 private List<CaptureRequest> createBurstRequest(CameraDevice cameraDevice, 509 List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, 510 Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap) { 511 CaptureRequest.Builder requestBuilder; 512 ArrayList<CaptureRequest> ret = new ArrayList<>(); 513 for (CaptureStageImpl captureStage : captureStageList) { 514 try { 515 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); 516 } catch (CameraAccessException e) { 517 return null; 518 } 519 520 // This will guarantee that client configured 521 // parameters always have the highest priority. 522 for (CaptureRequest.Key requestKey : mSupportedRequestKeys){ 523 Object value = clientRequest.get(requestKey); 524 if (value != null) { 525 captureStage.parameters.set(requestKey, value); 526 } 527 } 528 529 requestBuilder.addTarget(target); 530 CaptureRequest request = requestBuilder.build(); 531 CameraMetadataNative.update(request.getNativeMetadata(), captureStage.parameters); 532 ret.add(request); 533 if (captureMap != null) { 534 captureMap.put(request, captureStage.id); 535 } 536 } 537 538 return ret; 539 } 540 createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate, CaptureRequest clientRequest)541 private CaptureRequest createRequest(CameraDevice cameraDevice, 542 List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate, 543 CaptureRequest clientRequest) throws CameraAccessException { 544 CaptureRequest.Builder requestBuilder; 545 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); 546 if (target != null) { 547 requestBuilder.addTarget(target); 548 } 549 550 CaptureRequest ret = requestBuilder.build(); 551 CameraMetadataNative nativeMeta = ret.getNativeMetadata(); 552 for (CaptureStageImpl captureStage : captureStageList) { 553 if (captureStage != null) { 554 CameraMetadataNative.update(nativeMeta, captureStage.parameters); 555 } 556 } 557 558 if (clientRequest != null) { 559 // This will guarantee that client configured 560 // parameters always have the highest priority. 561 for (CaptureRequest.Key requestKey : mSupportedRequestKeys) { 562 Object value = clientRequest.get(requestKey); 563 if (value != null) { 564 nativeMeta.set(requestKey, value); 565 } 566 } 567 } 568 569 return ret; 570 } 571 createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate)572 private CaptureRequest createRequest(CameraDevice cameraDevice, 573 List<CaptureStageImpl> captureStageList, 574 Surface target, 575 int captureTemplate) throws CameraAccessException { 576 return createRequest(cameraDevice, captureStageList, target, captureTemplate, 577 /*clientRequest*/ null); 578 } 579 580 @Override capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)581 public int capture(@NonNull CaptureRequest request, 582 @NonNull Executor executor, 583 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 584 if (!mInitialized) { 585 throw new IllegalStateException("Uninitialized component"); 586 } 587 588 if (request.getTargets().size() != 1) { 589 throw new IllegalArgumentException("Single capture to both preview & still capture " + 590 "outputs target is not supported!"); 591 } 592 593 int seqId = -1; 594 if ((mClientCaptureSurface != null) && request.containsTarget(mClientCaptureSurface)) { 595 HashMap<CaptureRequest, Integer> requestMap = new HashMap<>(); 596 List<CaptureRequest> burstRequest; 597 try { 598 burstRequest = createBurstRequest(mCameraDevice, 599 mImageExtender.getCaptureStages(), request, mCameraBurstSurface, 600 CameraDevice.TEMPLATE_STILL_CAPTURE, requestMap); 601 } catch (RemoteException e) { 602 Log.e(TAG, "Failed to initialize internal burst request! Extension service does" 603 + " not respond!"); 604 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 605 } 606 if (burstRequest == null) { 607 throw new UnsupportedOperationException( 608 "Failed to create still capture burst request"); 609 } 610 611 seqId = mCaptureSession.captureBurstRequests(burstRequest, 612 new CameraExtensionUtils.HandlerExecutor(mHandler), 613 new BurstRequestHandler(request, executor, listener, requestMap, 614 mBurstCaptureImageCallback)); 615 } else if ((mClientRepeatingRequestSurface != null) && 616 request.containsTarget(mClientRepeatingRequestSurface)) { 617 618 CaptureRequest captureRequest = null; 619 try { 620 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 621 captureStageList.add(mPreviewExtender.getCaptureStage()); 622 623 captureRequest = createRequest(mCameraDevice, captureStageList, 624 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, request); 625 } catch (RemoteException e) { 626 Log.e(TAG, "Failed to initialize capture request! Extension service does" 627 + " not respond!"); 628 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 629 } 630 631 seqId = mCaptureSession.capture(captureRequest, new PreviewRequestHandler(request, 632 executor, listener, mRepeatingRequestImageCallback, true /*singleCapture*/), 633 mHandler); 634 } else { 635 throw new IllegalArgumentException("Capture request to unknown output surface!"); 636 } 637 638 return seqId; 639 } 640 641 @Override stopRepeating()642 public void stopRepeating() throws CameraAccessException { 643 synchronized (mInterfaceLock) { 644 if (!mInitialized) { 645 throw new IllegalStateException("Uninitialized component"); 646 } 647 648 mInternalRepeatingRequestEnabled = true; 649 mCaptureSession.stopRepeating(); 650 } 651 } 652 653 @Override close()654 public void close() throws CameraAccessException { 655 synchronized (mInterfaceLock) { 656 if (mInitialized) { 657 mInternalRepeatingRequestEnabled = false; 658 mCaptureSession.stopRepeating(); 659 660 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 661 try { 662 CaptureStageImpl disableParams = mPreviewExtender.onDisableSession(); 663 if (disableParams != null) { 664 captureStageList.add(disableParams); 665 } 666 667 CaptureStageImpl disableStillCaptureParams = 668 mImageExtender.onDisableSession(); 669 if (disableStillCaptureParams != null) { 670 captureStageList.add(disableStillCaptureParams); 671 } 672 } catch (RemoteException e) { 673 Log.e(TAG, "Failed to disable extension! Extension service does not " 674 + "respond!"); 675 } 676 if (!captureStageList.isEmpty()) { 677 CaptureRequest disableRequest = createRequest(mCameraDevice, captureStageList, 678 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); 679 mCaptureSession.capture(disableRequest, 680 new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler); 681 } 682 683 mCaptureSession.close(); 684 } 685 } 686 } 687 setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, InitialRequestHandler requestHandler)688 private void setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, 689 InitialRequestHandler requestHandler) 690 throws CameraAccessException { 691 CaptureRequest initialRequest = createRequest(mCameraDevice, 692 captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); 693 mCaptureSession.capture(initialRequest, requestHandler, mHandler); 694 } 695 setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler)696 private int setRepeatingRequest(CaptureStageImpl captureStage, 697 CameraCaptureSession.CaptureCallback requestHandler) throws CameraAccessException { 698 return setRepeatingRequest(captureStage, requestHandler, /*clientRequest*/ null); 699 } 700 setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest)701 private int setRepeatingRequest(CaptureStageImpl captureStage, 702 CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest) 703 throws CameraAccessException { 704 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 705 captureStageList.add(captureStage); 706 CaptureRequest repeatingRequest = createRequest(mCameraDevice, captureStageList, 707 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, clientRequest); 708 return mCaptureSession.setSingleRepeatingRequest(repeatingRequest, 709 new CameraExtensionUtils.HandlerExecutor(mHandler), requestHandler); 710 } 711 712 /** @hide */ release(boolean skipCloseNotification)713 public void release(boolean skipCloseNotification) { 714 boolean notifyClose = false; 715 716 synchronized (mInterfaceLock) { 717 mInternalRepeatingRequestEnabled = false; 718 mHandlerThread.quitSafely(); 719 720 try { 721 mPreviewExtender.onDeInit(); 722 mImageExtender.onDeInit(); 723 } catch (RemoteException e) { 724 Log.e(TAG, "Failed to release extensions! Extension service does not" 725 + " respond!"); 726 } 727 728 if (mExtensionClientId >= 0) { 729 CameraExtensionCharacteristics.unregisterClient(mExtensionClientId); 730 if (mInitialized) { 731 notifyClose = true; 732 CameraExtensionCharacteristics.releaseSession(); 733 } 734 } 735 mInitialized = false; 736 737 if (mRepeatingRequestImageCallback != null) { 738 mRepeatingRequestImageCallback.close(); 739 mRepeatingRequestImageCallback = null; 740 } 741 742 if (mRepeatingRequestImageReader != null) { 743 mRepeatingRequestImageReader.close(); 744 mRepeatingRequestImageReader = null; 745 } 746 747 if (mBurstCaptureImageCallback != null) { 748 mBurstCaptureImageCallback.close(); 749 mBurstCaptureImageCallback = null; 750 } 751 752 if (mBurstCaptureImageReader != null) { 753 mBurstCaptureImageReader.close(); 754 mBurstCaptureImageReader = null; 755 } 756 757 if (mStubCaptureImageReader != null) { 758 mStubCaptureImageReader.close(); 759 mStubCaptureImageReader = null; 760 } 761 762 if (mRepeatingRequestImageWriter != null) { 763 mRepeatingRequestImageWriter.close(); 764 mRepeatingRequestImageWriter = null; 765 } 766 767 if (mPreviewImageProcessor != null) { 768 mPreviewImageProcessor.close(); 769 mPreviewImageProcessor = null; 770 } 771 772 if (mImageJpegProcessor != null) { 773 mImageJpegProcessor.close(); 774 mImageJpegProcessor = null; 775 } 776 777 mCaptureSession = null; 778 mImageProcessor = null; 779 mCameraRepeatingSurface = mClientRepeatingRequestSurface = null; 780 mCameraBurstSurface = mClientCaptureSurface = null; 781 } 782 783 if (notifyClose && !skipCloseNotification) { 784 final long ident = Binder.clearCallingIdentity(); 785 try { 786 mExecutor.execute(() -> mCallbacks.onClosed(CameraExtensionSessionImpl.this)); 787 } finally { 788 Binder.restoreCallingIdentity(ident); 789 } 790 } 791 } 792 notifyConfigurationFailure()793 private void notifyConfigurationFailure() { 794 synchronized (mInterfaceLock) { 795 if (mInitialized) { 796 return; 797 } 798 } 799 800 release(true /*skipCloseNotification*/); 801 802 final long ident = Binder.clearCallingIdentity(); 803 try { 804 mExecutor.execute( 805 () -> mCallbacks.onConfigureFailed(CameraExtensionSessionImpl.this)); 806 } finally { 807 Binder.restoreCallingIdentity(ident); 808 } 809 } 810 notifyConfigurationSuccess()811 private void notifyConfigurationSuccess() { 812 synchronized (mInterfaceLock) { 813 if (mInitialized) { 814 return; 815 } else { 816 mInitialized = true; 817 } 818 } 819 820 final long ident = Binder.clearCallingIdentity(); 821 try { 822 mExecutor.execute(() -> mCallbacks.onConfigured(CameraExtensionSessionImpl.this)); 823 } finally { 824 Binder.restoreCallingIdentity(ident); 825 } 826 } 827 828 private class SessionStateHandler extends 829 android.hardware.camera2.CameraCaptureSession.StateCallback { 830 @Override onClosed(@onNull CameraCaptureSession session)831 public void onClosed(@NonNull CameraCaptureSession session) { 832 release(false /*skipCloseNotification*/); 833 } 834 835 @Override onConfigureFailed(@onNull CameraCaptureSession session)836 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 837 notifyConfigurationFailure(); 838 } 839 840 @Override onConfigured(@onNull CameraCaptureSession session)841 public void onConfigured(@NonNull CameraCaptureSession session) { 842 synchronized (mInterfaceLock) { 843 mCaptureSession = session; 844 try { 845 finishPipelineInitialization(); 846 CameraExtensionCharacteristics.initializeSession(mInitializeHandler); 847 } catch (RemoteException e) { 848 Log.e(TAG, "Failed to initialize session! Extension service does" 849 + " not respond!"); 850 notifyConfigurationFailure(); 851 } 852 } 853 } 854 } 855 856 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 857 @Override onSuccess()858 public void onSuccess() { 859 boolean status = true; 860 ArrayList<CaptureStageImpl> initialRequestList = 861 compileInitialRequestList(); 862 if (!initialRequestList.isEmpty()) { 863 try { 864 setInitialCaptureRequest(initialRequestList, 865 new InitialRequestHandler( 866 mRepeatingRequestImageCallback)); 867 } catch (CameraAccessException e) { 868 Log.e(TAG, 869 "Failed to initialize the initial capture " 870 + "request!"); 871 status = false; 872 } 873 } else { 874 try { 875 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 876 new PreviewRequestHandler(null, null, null, 877 mRepeatingRequestImageCallback)); 878 } catch (CameraAccessException | RemoteException e) { 879 Log.e(TAG, 880 "Failed to initialize internal repeating " 881 + "request!"); 882 status = false; 883 } 884 885 } 886 887 if (!status) { 888 notifyConfigurationFailure(); 889 } 890 } 891 892 @Override onFailure()893 public void onFailure() { 894 mCaptureSession.close(); 895 Log.e(TAG, "Failed to initialize proxy service session!" 896 + " This can happen when trying to configure multiple " 897 + "concurrent extension sessions!"); 898 notifyConfigurationFailure(); 899 } 900 } 901 902 private class BurstRequestHandler extends CameraCaptureSession.CaptureCallback { 903 private final Executor mExecutor; 904 private final ExtensionCaptureCallback mCallbacks; 905 private final CaptureRequest mClientRequest; 906 private final HashMap<CaptureRequest, Integer> mCaptureRequestMap; 907 private final CameraOutputImageCallback mBurstImageCallback; 908 909 private HashMap<Integer, Pair<Image, TotalCaptureResult>> mCaptureStageMap = 910 new HashMap<>(); 911 private LongSparseArray<Pair<Image, Integer>> mCapturePendingMap = 912 new LongSparseArray<>(); 913 914 private ImageCallback mImageCallback = null; 915 private boolean mCaptureFailed = false; 916 private CaptureResultHandler mCaptureResultHandler = null; 917 BurstRequestHandler(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback callbacks, @NonNull HashMap<CaptureRequest, Integer> requestMap, @Nullable CameraOutputImageCallback imageCallback)918 public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor, 919 @NonNull ExtensionCaptureCallback callbacks, 920 @NonNull HashMap<CaptureRequest, Integer> requestMap, 921 @Nullable CameraOutputImageCallback imageCallback) { 922 mClientRequest = request; 923 mExecutor = executor; 924 mCallbacks = callbacks; 925 mCaptureRequestMap = requestMap; 926 mBurstImageCallback = imageCallback; 927 } 928 notifyCaptureFailed()929 private void notifyCaptureFailed() { 930 if (!mCaptureFailed) { 931 mCaptureFailed = true; 932 933 final long ident = Binder.clearCallingIdentity(); 934 try { 935 mExecutor.execute( 936 () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, 937 mClientRequest)); 938 } finally { 939 Binder.restoreCallingIdentity(ident); 940 } 941 942 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) { 943 captureStage.first.close(); 944 } 945 mCaptureStageMap.clear(); 946 } 947 } 948 949 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)950 public void onCaptureStarted(@NonNull CameraCaptureSession session, 951 @NonNull CaptureRequest request, 952 long timestamp, 953 long frameNumber) { 954 // Trigger the client callback only once in case of burst request 955 boolean initialCallback = false; 956 synchronized (mInterfaceLock) { 957 if ((mImageProcessor != null) && (mImageCallback == null)) { 958 mImageCallback = new ImageCallback(); 959 initialCallback = true; 960 } else if (mImageProcessor == null) { 961 // No burst expected in this case 962 initialCallback = true; 963 } 964 } 965 966 if (initialCallback) { 967 final long ident = Binder.clearCallingIdentity(); 968 try { 969 mExecutor.execute( 970 () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, 971 mClientRequest, timestamp)); 972 } finally { 973 Binder.restoreCallingIdentity(ident); 974 } 975 } 976 977 if ((mBurstImageCallback != null) && (mImageCallback != null)) { 978 mBurstImageCallback.registerListener(timestamp, mImageCallback); 979 } 980 } 981 982 @Override onCaptureBufferLost(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull Surface target, long frameNumber)983 public void onCaptureBufferLost(@NonNull CameraCaptureSession session, 984 @NonNull CaptureRequest request, 985 @NonNull Surface target, long frameNumber) { 986 notifyCaptureFailed(); 987 } 988 989 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)990 public void onCaptureFailed(@NonNull CameraCaptureSession session, 991 @NonNull CaptureRequest request, 992 @NonNull CaptureFailure failure) { 993 notifyCaptureFailed(); 994 } 995 996 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)997 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 998 int sequenceId) { 999 final long ident = Binder.clearCallingIdentity(); 1000 try { 1001 mExecutor.execute( 1002 () -> mCallbacks.onCaptureSequenceAborted(CameraExtensionSessionImpl.this, 1003 sequenceId)); 1004 } finally { 1005 Binder.restoreCallingIdentity(ident); 1006 } 1007 } 1008 1009 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1010 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1011 int sequenceId, 1012 long frameNumber) { 1013 final long ident = Binder.clearCallingIdentity(); 1014 try { 1015 mExecutor.execute( 1016 () -> mCallbacks 1017 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, 1018 sequenceId)); 1019 } finally { 1020 Binder.restoreCallingIdentity(ident); 1021 } 1022 } 1023 1024 @Override onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1025 public void onCaptureCompleted(@NonNull CameraCaptureSession session, 1026 @NonNull CaptureRequest request, 1027 @NonNull TotalCaptureResult result) { 1028 if (!mCaptureRequestMap.containsKey(request)) { 1029 Log.e(TAG, 1030 "Unexpected still capture request received!"); 1031 return; 1032 } 1033 Integer stageId = mCaptureRequestMap.get(request); 1034 1035 Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1036 if (timestamp != null) { 1037 if (mCaptureResultsSupported && (mCaptureResultHandler == null)) { 1038 mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, 1039 mCallbacks, result.getSequenceId()); 1040 } 1041 if (mImageProcessor != null) { 1042 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { 1043 Image img = mCapturePendingMap.get(timestamp).first; 1044 mCaptureStageMap.put(stageId, new Pair<>(img, result)); 1045 checkAndFireBurstProcessing(); 1046 } else { 1047 mCapturePendingMap.put(timestamp, new Pair<>(null, stageId)); 1048 mCaptureStageMap.put(stageId, new Pair<>(null, result)); 1049 } 1050 } else { 1051 mCaptureRequestMap.clear(); 1052 final long ident = Binder.clearCallingIdentity(); 1053 try { 1054 mExecutor.execute( 1055 () -> mCallbacks 1056 .onCaptureProcessStarted(CameraExtensionSessionImpl.this, 1057 mClientRequest)); 1058 1059 if (mCaptureResultHandler != null) { 1060 mCaptureResultHandler.onCaptureCompleted(timestamp, 1061 initializeFilteredResults(result)); 1062 } 1063 } finally { 1064 Binder.restoreCallingIdentity(ident); 1065 } 1066 } 1067 } else { 1068 Log.e(TAG, 1069 "Capture result without valid sensor timestamp!"); 1070 } 1071 } 1072 checkAndFireBurstProcessing()1073 private void checkAndFireBurstProcessing() { 1074 if (mCaptureRequestMap.size() == mCaptureStageMap.size()) { 1075 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap 1076 .values()) { 1077 if ((captureStage.first == null) || (captureStage.second == null)) { 1078 return; 1079 } 1080 } 1081 1082 mCaptureRequestMap.clear(); 1083 mCapturePendingMap.clear(); 1084 boolean processStatus = true; 1085 Byte jpegQuality = mClientRequest.get(CaptureRequest.JPEG_QUALITY); 1086 Integer jpegOrientation = mClientRequest.get(CaptureRequest.JPEG_ORIENTATION); 1087 List<CaptureBundle> captureList = initializeParcelable(mCaptureStageMap, 1088 jpegOrientation, jpegQuality); 1089 try { 1090 mImageProcessor.process(captureList, mCaptureResultHandler); 1091 } catch (RemoteException e) { 1092 Log.e(TAG, "Failed to process multi-frame request! Extension service " 1093 + "does not respond!"); 1094 processStatus = false; 1095 } 1096 1097 for (CaptureBundle bundle : captureList) { 1098 bundle.captureImage.buffer.close(); 1099 } 1100 captureList.clear(); 1101 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) { 1102 captureStage.first.close(); 1103 } 1104 mCaptureStageMap.clear(); 1105 1106 final long ident = Binder.clearCallingIdentity(); 1107 try { 1108 if (processStatus) { 1109 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( 1110 CameraExtensionSessionImpl.this, mClientRequest)); 1111 } else { 1112 mExecutor.execute(() -> mCallbacks.onCaptureFailed( 1113 CameraExtensionSessionImpl.this, mClientRequest)); 1114 } 1115 } finally { 1116 Binder.restoreCallingIdentity(ident); 1117 } 1118 } 1119 } 1120 1121 private class ImageCallback implements OnImageAvailableListener { 1122 @Override onImageDropped(long timestamp)1123 public void onImageDropped(long timestamp) { 1124 notifyCaptureFailed(); 1125 } 1126 1127 @Override onImageAvailable(ImageReader reader, Image img)1128 public void onImageAvailable(ImageReader reader, Image img) { 1129 if (mCaptureFailed) { 1130 img.close(); 1131 } 1132 1133 long timestamp = img.getTimestamp(); 1134 reader.detachImage(img); 1135 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { 1136 Integer stageId = mCapturePendingMap.get(timestamp).second; 1137 Pair<Image, TotalCaptureResult> captureStage = 1138 mCaptureStageMap.get(stageId); 1139 if (captureStage != null) { 1140 mCaptureStageMap.put(stageId, 1141 new Pair<>(img, 1142 captureStage.second)); 1143 checkAndFireBurstProcessing(); 1144 } else { 1145 Log.e(TAG, 1146 "Capture stage: " + 1147 mCapturePendingMap.get(timestamp).second + 1148 " is absent!"); 1149 } 1150 } else { 1151 mCapturePendingMap.put(timestamp, 1152 new Pair<>(img, 1153 -1)); 1154 } 1155 } 1156 } 1157 } 1158 1159 private class ImageLoopbackCallback implements OnImageAvailableListener { 1160 @Override onImageDropped(long timestamp)1161 public void onImageDropped(long timestamp) { } 1162 1163 @Override onImageAvailable(ImageReader reader, Image img)1164 public void onImageAvailable(ImageReader reader, Image img) { 1165 img.close(); 1166 } 1167 } 1168 1169 private class InitialRequestHandler extends CameraCaptureSession.CaptureCallback { 1170 private final CameraOutputImageCallback mImageCallback; 1171 InitialRequestHandler(CameraOutputImageCallback imageCallback)1172 public InitialRequestHandler(CameraOutputImageCallback imageCallback) { 1173 mImageCallback = imageCallback; 1174 } 1175 1176 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1177 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1178 @NonNull CaptureRequest request, long timestamp, long frameNumber) { 1179 mImageCallback.registerListener(timestamp, new ImageLoopbackCallback()); 1180 } 1181 1182 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1183 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 1184 int sequenceId) { 1185 Log.e(TAG, "Initial capture request aborted!"); 1186 notifyConfigurationFailure(); 1187 } 1188 1189 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1190 public void onCaptureFailed(@NonNull CameraCaptureSession session, 1191 @NonNull CaptureRequest request, 1192 @NonNull CaptureFailure failure) { 1193 Log.e(TAG, "Initial capture request failed!"); 1194 notifyConfigurationFailure(); 1195 } 1196 1197 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1198 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1199 int sequenceId, 1200 long frameNumber) { 1201 boolean status = true; 1202 synchronized (mInterfaceLock) { 1203 /** 1204 * Initialize and set the initial repeating request which will execute in the 1205 * absence of client repeating requests. 1206 */ 1207 try { 1208 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 1209 new PreviewRequestHandler(null, null, null, 1210 mImageCallback)); 1211 } catch (CameraAccessException | RemoteException e) { 1212 Log.e(TAG, "Failed to start the internal repeating request!"); 1213 status = false; 1214 } 1215 1216 } 1217 1218 if (!status) { 1219 notifyConfigurationFailure(); 1220 } 1221 } 1222 } 1223 1224 private interface OnImageAvailableListener { onImageDropped(long timestamp)1225 void onImageDropped(long timestamp); onImageAvailable(ImageReader reader, Image img)1226 void onImageAvailable (ImageReader reader, Image img); 1227 } 1228 1229 private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener, 1230 Closeable { 1231 private final ImageReader mImageReader; 1232 // Map timestamp to specific images and listeners 1233 private HashMap<Long, Pair<Image, OnImageAvailableListener>> mImageListenerMap = 1234 new HashMap<>(); 1235 private boolean mOutOfBuffers = false; 1236 CameraOutputImageCallback(ImageReader imageReader)1237 CameraOutputImageCallback(ImageReader imageReader) { 1238 mImageReader = imageReader; 1239 } 1240 1241 @Override onImageAvailable(ImageReader reader)1242 public void onImageAvailable(ImageReader reader) { 1243 Image img; 1244 try { 1245 img = reader.acquireNextImage(); 1246 } catch (IllegalStateException e) { 1247 Log.e(TAG, "Failed to acquire image, too many images pending!"); 1248 mOutOfBuffers = true; 1249 return; 1250 } 1251 if (img == null) { 1252 Log.e(TAG, "Invalid image!"); 1253 return; 1254 } 1255 1256 Long timestamp = img.getTimestamp(); 1257 if (mImageListenerMap.containsKey(timestamp)) { 1258 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(timestamp); 1259 if (entry.second != null) { 1260 entry.second.onImageAvailable(reader, img); 1261 } else { 1262 Log.w(TAG, "Invalid image listener, dropping frame!"); 1263 img.close(); 1264 } 1265 } else { 1266 mImageListenerMap.put(img.getTimestamp(), new Pair<>(img, null)); 1267 } 1268 1269 notifyDroppedImages(timestamp); 1270 } 1271 notifyDroppedImages(long timestamp)1272 private void notifyDroppedImages(long timestamp) { 1273 Set<Long> timestamps = mImageListenerMap.keySet(); 1274 ArrayList<Long> removedTs = new ArrayList<>(); 1275 for (long ts : timestamps) { 1276 if (ts < timestamp) { 1277 Log.e(TAG, "Dropped image with ts: " + ts); 1278 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts); 1279 if (entry.second != null) { 1280 entry.second.onImageDropped(ts); 1281 } 1282 if (entry.first != null) { 1283 entry.first.close(); 1284 } 1285 removedTs.add(ts); 1286 } 1287 } 1288 for (long ts : removedTs) { 1289 mImageListenerMap.remove(ts); 1290 } 1291 } 1292 registerListener(Long timestamp, OnImageAvailableListener listener)1293 public void registerListener(Long timestamp, OnImageAvailableListener listener) { 1294 if (mImageListenerMap.containsKey(timestamp)) { 1295 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove(timestamp); 1296 if (entry.first != null) { 1297 listener.onImageAvailable(mImageReader, entry.first); 1298 if (mOutOfBuffers) { 1299 mOutOfBuffers = false; 1300 Log.w(TAG,"Out of buffers, retry!"); 1301 onImageAvailable(mImageReader); 1302 } 1303 } else { 1304 Log.w(TAG, "No valid image for listener with ts: " + 1305 timestamp.longValue()); 1306 } 1307 } else { 1308 mImageListenerMap.put(timestamp, new Pair<>(null, listener)); 1309 } 1310 } 1311 1312 @Override close()1313 public void close() { 1314 for (Pair<Image, OnImageAvailableListener> entry : mImageListenerMap.values()) { 1315 if (entry.first != null) { 1316 entry.first.close(); 1317 } 1318 } 1319 for (long timestamp : mImageListenerMap.keySet()) { 1320 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp); 1321 if (entry.second != null) { 1322 entry.second.onImageDropped(timestamp); 1323 } 1324 } 1325 mImageListenerMap.clear(); 1326 } 1327 } 1328 1329 private class CloseRequestHandler extends CameraCaptureSession.CaptureCallback { 1330 private final CameraOutputImageCallback mImageCallback; 1331 CloseRequestHandler(CameraOutputImageCallback imageCallback)1332 public CloseRequestHandler(CameraOutputImageCallback imageCallback) { 1333 mImageCallback = imageCallback; 1334 } 1335 1336 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1337 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1338 @NonNull CaptureRequest request, long timestamp, long frameNumber) { 1339 mImageCallback.registerListener(timestamp, new ImageLoopbackCallback()); 1340 } 1341 } 1342 1343 private class CaptureResultHandler extends IProcessResultImpl.Stub { 1344 private final Executor mExecutor; 1345 private final ExtensionCaptureCallback mCallbacks; 1346 private final CaptureRequest mClientRequest; 1347 private final int mRequestId; 1348 CaptureResultHandler(@onNull CaptureRequest clientRequest, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, int requestId)1349 public CaptureResultHandler(@NonNull CaptureRequest clientRequest, 1350 @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, 1351 int requestId) { 1352 mClientRequest = clientRequest; 1353 mExecutor = executor; 1354 mCallbacks = listener; 1355 mRequestId = requestId; 1356 } 1357 1358 @Override onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result)1359 public void onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result) { 1360 if (result == null) { 1361 Log.e(TAG,"Invalid capture result!"); 1362 return; 1363 } 1364 1365 result.set(CaptureResult.SENSOR_TIMESTAMP, shutterTimestamp); 1366 TotalCaptureResult totalResult = new TotalCaptureResult(mCameraDevice.getId(), result, 1367 mClientRequest, mRequestId, shutterTimestamp, new ArrayList<CaptureResult>(), 1368 mSessionId, new PhysicalCaptureResultInfo[0]); 1369 final long ident = Binder.clearCallingIdentity(); 1370 try { 1371 mExecutor.execute( 1372 () -> mCallbacks.onCaptureResultAvailable(CameraExtensionSessionImpl.this, 1373 mClientRequest, totalResult)); 1374 } finally { 1375 Binder.restoreCallingIdentity(ident); 1376 } 1377 } 1378 } 1379 1380 // This handler can operate in three modes: 1381 // 1) Using valid client callbacks, which means camera buffers will be propagated the 1382 // registered output surfaces and clients will be notified accordingly. 1383 // 2) Without any client callbacks where an internal repeating request is kept active 1384 // to satisfy the extensions continuous preview/(repeating request) requirement. 1385 // 3) Single capture mode, where internal repeating requests are ignored and the preview 1386 // logic is only triggered for the image processor case. 1387 private class PreviewRequestHandler extends CameraCaptureSession.CaptureCallback { 1388 private final Executor mExecutor; 1389 private final ExtensionCaptureCallback mCallbacks; 1390 private final CaptureRequest mClientRequest; 1391 private final boolean mClientNotificationsEnabled; 1392 private final CameraOutputImageCallback mRepeatingImageCallback; 1393 private final boolean mSingleCapture; 1394 private OnImageAvailableListener mImageCallback = null; 1395 private LongSparseArray<Pair<Image, TotalCaptureResult>> mPendingResultMap = 1396 new LongSparseArray<>(); 1397 private CaptureResultHandler mCaptureResultHandler = null; 1398 1399 private boolean mRequestUpdatedNeeded = false; 1400 PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback)1401 public PreviewRequestHandler(@Nullable CaptureRequest clientRequest, 1402 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, 1403 @NonNull CameraOutputImageCallback imageCallback) { 1404 this(clientRequest, executor, listener, imageCallback, false /*singleCapture*/); 1405 } 1406 PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture)1407 public PreviewRequestHandler(@Nullable CaptureRequest clientRequest, 1408 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, 1409 @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture) { 1410 mClientRequest = clientRequest; 1411 mExecutor = executor; 1412 mCallbacks = listener; 1413 mClientNotificationsEnabled = 1414 (mClientRequest != null) && (mExecutor != null) && (mCallbacks != null); 1415 mRepeatingImageCallback = imageCallback; 1416 mSingleCapture = singleCapture; 1417 } 1418 1419 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1420 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1421 @NonNull CaptureRequest request, 1422 long timestamp, 1423 long frameNumber) { 1424 synchronized (mInterfaceLock) { 1425 // Setup the image callback handler for this repeating request just once 1426 // after streaming resumes. 1427 if (mImageCallback == null) { 1428 if (mPreviewProcessorType == 1429 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { 1430 if (mClientNotificationsEnabled) { 1431 mPreviewImageProcessor.onOutputSurface(mClientRepeatingRequestSurface, 1432 nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); 1433 } else { 1434 mPreviewImageProcessor.onOutputSurface(null, -1); 1435 } 1436 mImageCallback = new ImageProcessCallback(); 1437 } else { 1438 mImageCallback = mClientNotificationsEnabled ? 1439 new ImageForwardCallback(mRepeatingRequestImageWriter) : 1440 new ImageLoopbackCallback(); 1441 } 1442 } 1443 } 1444 1445 if (mClientNotificationsEnabled) { 1446 final long ident = Binder.clearCallingIdentity(); 1447 try { 1448 mExecutor.execute( 1449 () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, 1450 mClientRequest, timestamp)); 1451 } finally { 1452 Binder.restoreCallingIdentity(ident); 1453 } 1454 } 1455 1456 mRepeatingImageCallback.registerListener(timestamp, mImageCallback); 1457 } 1458 1459 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1460 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 1461 int sequenceId) { 1462 synchronized (mInterfaceLock) { 1463 if (mInternalRepeatingRequestEnabled && !mSingleCapture) { 1464 resumeInternalRepeatingRequest(true); 1465 } 1466 } 1467 1468 if (mClientNotificationsEnabled) { 1469 final long ident = Binder.clearCallingIdentity(); 1470 try { 1471 mExecutor.execute( 1472 () -> mCallbacks 1473 .onCaptureSequenceAborted(CameraExtensionSessionImpl.this, 1474 sequenceId)); 1475 } finally { 1476 Binder.restoreCallingIdentity(ident); 1477 } 1478 } else { 1479 notifyConfigurationFailure(); 1480 } 1481 } 1482 1483 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1484 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1485 int sequenceId, 1486 long frameNumber) { 1487 1488 synchronized (mInterfaceLock) { 1489 if (mRequestUpdatedNeeded && !mSingleCapture) { 1490 mRequestUpdatedNeeded = false; 1491 resumeInternalRepeatingRequest(false); 1492 } else if (mInternalRepeatingRequestEnabled && !mSingleCapture) { 1493 resumeInternalRepeatingRequest(true); 1494 } 1495 } 1496 1497 if (mClientNotificationsEnabled) { 1498 final long ident = Binder.clearCallingIdentity(); 1499 try { 1500 mExecutor.execute( 1501 () -> mCallbacks 1502 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, 1503 sequenceId)); 1504 } finally { 1505 Binder.restoreCallingIdentity(ident); 1506 } 1507 } 1508 } 1509 1510 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1511 public void onCaptureFailed(@NonNull CameraCaptureSession session, 1512 @NonNull CaptureRequest request, 1513 @NonNull CaptureFailure failure) { 1514 1515 if (mClientNotificationsEnabled) { 1516 final long ident = Binder.clearCallingIdentity(); 1517 try { 1518 mExecutor.execute( 1519 () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, 1520 mClientRequest)); 1521 } finally { 1522 Binder.restoreCallingIdentity(ident); 1523 } 1524 } 1525 } 1526 1527 @Override onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1528 public void onCaptureCompleted(@NonNull CameraCaptureSession session, 1529 @NonNull CaptureRequest request, 1530 @NonNull TotalCaptureResult result) { 1531 boolean notifyClient = mClientNotificationsEnabled; 1532 boolean processStatus = true; 1533 1534 synchronized (mInterfaceLock) { 1535 final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1536 if (timestamp != null) { 1537 if (mCaptureResultsSupported && mClientNotificationsEnabled && 1538 (mCaptureResultHandler == null)) { 1539 mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, 1540 mCallbacks, result.getSequenceId()); 1541 } 1542 if ((!mSingleCapture) && (mPreviewProcessorType == 1543 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY)) { 1544 CaptureStageImpl captureStage = null; 1545 try { 1546 captureStage = mPreviewRequestUpdateProcessor.process( 1547 result.getNativeMetadata(), result.getSequenceId()); 1548 } catch (RemoteException e) { 1549 Log.e(TAG, "Extension service does not respond during " + 1550 "processing!"); 1551 } 1552 if (captureStage != null) { 1553 try { 1554 setRepeatingRequest(captureStage, this, request); 1555 mRequestUpdatedNeeded = true; 1556 } catch (IllegalStateException e) { 1557 // This is possible in case the camera device closes and the 1558 // and the callback here is executed before the onClosed 1559 // notification. 1560 } catch (CameraAccessException e) { 1561 Log.e(TAG, "Failed to update repeating request settings!"); 1562 } 1563 } else { 1564 mRequestUpdatedNeeded = false; 1565 } 1566 } else if (mPreviewProcessorType == 1567 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { 1568 int idx = mPendingResultMap.indexOfKey(timestamp); 1569 1570 if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) { 1571 // Image was dropped before we can receive the capture results 1572 if ((mCaptureResultHandler != null)) { 1573 mCaptureResultHandler.onCaptureCompleted(timestamp, 1574 initializeFilteredResults(result)); 1575 } 1576 discardPendingRepeatingResults(idx, mPendingResultMap, false); 1577 } else if (idx >= 0) { 1578 // Image came before the capture results 1579 ParcelImage parcelImage = initializeParcelImage( 1580 mPendingResultMap.get(timestamp).first); 1581 try { 1582 mPreviewImageProcessor.process(parcelImage, result, 1583 mCaptureResultHandler); 1584 } catch (RemoteException e) { 1585 processStatus = false; 1586 Log.e(TAG, "Extension service does not respond during " + 1587 "processing, dropping frame!"); 1588 } catch (RuntimeException e) { 1589 // Runtime exceptions can happen in a few obscure cases where the 1590 // client tries to initialize a new capture session while this 1591 // session is still ongoing. In such scenario, the camera will 1592 // disconnect from the intermediate output surface, which will 1593 // invalidate the images that we acquired previously. This can 1594 // happen before we get notified via "onClosed" so there aren't 1595 // many options to avoid the exception. 1596 processStatus = false; 1597 Log.e(TAG, "Runtime exception encountered during buffer " + 1598 "processing, dropping frame!"); 1599 } finally { 1600 parcelImage.buffer.close(); 1601 mPendingResultMap.get(timestamp).first.close(); 1602 } 1603 discardPendingRepeatingResults(idx, mPendingResultMap, false); 1604 } else { 1605 // Image not yet available 1606 notifyClient = false; 1607 mPendingResultMap.put(timestamp, 1608 new Pair<>(null, 1609 result)); 1610 } 1611 } else { 1612 // No special handling for PROCESSOR_TYPE_NONE 1613 } 1614 if (notifyClient) { 1615 final long ident = Binder.clearCallingIdentity(); 1616 try { 1617 if (processStatus) { 1618 mExecutor.execute(() -> mCallbacks 1619 .onCaptureProcessStarted( 1620 CameraExtensionSessionImpl.this, 1621 mClientRequest)); 1622 if ((mCaptureResultHandler != null) && (mPreviewProcessorType != 1623 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) { 1624 mCaptureResultHandler.onCaptureCompleted(timestamp, 1625 initializeFilteredResults(result)); 1626 } 1627 } else { 1628 mExecutor.execute( 1629 () -> mCallbacks 1630 .onCaptureFailed( 1631 CameraExtensionSessionImpl.this, 1632 mClientRequest)); 1633 } 1634 } finally { 1635 Binder.restoreCallingIdentity(ident); 1636 } 1637 } 1638 } else { 1639 Log.e(TAG, 1640 "Result without valid sensor timestamp!"); 1641 } 1642 } 1643 1644 if (!notifyClient) { 1645 notifyConfigurationSuccess(); 1646 } 1647 } 1648 resumeInternalRepeatingRequest(boolean internal)1649 private void resumeInternalRepeatingRequest(boolean internal) { 1650 try { 1651 if (internal) { 1652 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 1653 new PreviewRequestHandler(null, null, null, 1654 mRepeatingImageCallback)); 1655 } else { 1656 setRepeatingRequest(mPreviewExtender.getCaptureStage(), this, mClientRequest); 1657 } 1658 } catch (RemoteException e) { 1659 Log.e(TAG, "Failed to resume internal repeating request, extension service" 1660 + " fails to respond!"); 1661 } catch (IllegalStateException e) { 1662 // This is possible in case we try to resume before the state "onClosed" 1663 // notification is able to reach us. 1664 Log.w(TAG, "Failed to resume internal repeating request!"); 1665 } catch (CameraAccessException e) { 1666 Log.e(TAG, "Failed to resume internal repeating request!"); 1667 } 1668 } 1669 1670 // Find the timestamp of the oldest pending buffer calculatePruneThreshold( LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap)1671 private Long calculatePruneThreshold( 1672 LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap) { 1673 long oldestTimestamp = Long.MAX_VALUE; 1674 for (int idx = 0; idx < previewMap.size(); idx++) { 1675 Pair<Image, TotalCaptureResult> entry = previewMap.valueAt(idx); 1676 long timestamp = previewMap.keyAt(idx); 1677 if ((entry.first != null) && (timestamp < oldestTimestamp)) { 1678 oldestTimestamp = timestamp; 1679 } 1680 } 1681 return (oldestTimestamp == Long.MAX_VALUE) ? 0 : oldestTimestamp; 1682 } 1683 discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap, boolean notifyCurrentIndex)1684 private void discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, 1685 TotalCaptureResult>> previewMap, boolean notifyCurrentIndex) { 1686 if (idx < 0) { 1687 return; 1688 } 1689 for (int i = idx; i >= 0; i--) { 1690 if (previewMap.valueAt(i).first != null) { 1691 previewMap.valueAt(i).first.close(); 1692 } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) && 1693 ((i != idx) || notifyCurrentIndex)) { 1694 TotalCaptureResult result = previewMap.valueAt(i).second; 1695 Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1696 if (mCaptureResultHandler != null) { 1697 mCaptureResultHandler.onCaptureCompleted(timestamp, 1698 initializeFilteredResults(result)); 1699 } 1700 1701 Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i)); 1702 final long ident = Binder.clearCallingIdentity(); 1703 try { 1704 mExecutor.execute( 1705 () -> mCallbacks 1706 .onCaptureFailed(CameraExtensionSessionImpl.this, 1707 mClientRequest)); 1708 } finally { 1709 Binder.restoreCallingIdentity(ident); 1710 } 1711 1712 } 1713 previewMap.removeAt(i); 1714 } 1715 } 1716 1717 private class ImageForwardCallback implements OnImageAvailableListener { 1718 private final ImageWriter mOutputWriter; 1719 ImageForwardCallback(@onNull ImageWriter imageWriter)1720 public ImageForwardCallback(@NonNull ImageWriter imageWriter) { 1721 mOutputWriter = imageWriter; 1722 } 1723 1724 @Override onImageDropped(long timestamp)1725 public void onImageDropped(long timestamp) { 1726 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), 1727 mPendingResultMap, true); 1728 } 1729 1730 @Override onImageAvailable(ImageReader reader, Image img)1731 public void onImageAvailable(ImageReader reader, Image img) { 1732 if (img == null) { 1733 Log.e(TAG, "Invalid image!"); 1734 return; 1735 } 1736 1737 try { 1738 mOutputWriter.queueInputImage(img); 1739 } catch (IllegalStateException e) { 1740 // This is possible in case the client disconnects from the output surface 1741 // abruptly. 1742 Log.w(TAG, "Output surface likely abandoned, dropping buffer!"); 1743 img.close(); 1744 } catch (RuntimeException e) { 1745 // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage 1746 // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the 1747 // bug went unchecked for a few years and now its behavior cannot be changed 1748 // without breaking backwards compatibility. 1749 1750 if (!e.getClass().equals(RuntimeException.class)) { 1751 // re-throw any exceptions that aren't base RuntimeException since they are 1752 // coming from elsewhere, and we shouldn't silently drop those. 1753 throw e; 1754 } 1755 1756 Log.w(TAG, "Output surface likely abandoned, dropping buffer!"); 1757 img.close(); 1758 } 1759 } 1760 } 1761 1762 private class ImageProcessCallback implements OnImageAvailableListener { 1763 1764 @Override onImageDropped(long timestamp)1765 public void onImageDropped(long timestamp) { 1766 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), 1767 mPendingResultMap, true); 1768 // Add an empty frame&results entry to flag that we dropped a frame 1769 // and valid capture results can immediately return to client. 1770 mPendingResultMap.put(timestamp, new Pair<>(null, null)); 1771 } 1772 1773 @Override onImageAvailable(ImageReader reader, Image img)1774 public void onImageAvailable(ImageReader reader, Image img) { 1775 if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) { 1776 // We reached the maximum acquired images limit. This is possible in case we 1777 // have capture failures that result in absent or missing capture results. In 1778 // such scenario we can prune the oldest pending buffer. 1779 discardPendingRepeatingResults( 1780 mPendingResultMap 1781 .indexOfKey(calculatePruneThreshold(mPendingResultMap)), 1782 mPendingResultMap, true); 1783 } 1784 1785 if (img == null) { 1786 Log.e(TAG, 1787 "Invalid preview buffer!"); 1788 return; 1789 } 1790 try { 1791 reader.detachImage(img); 1792 } catch (IllegalStateException e) { 1793 Log.e(TAG, "Failed to detach image!"); 1794 img.close(); 1795 return; 1796 } catch (RuntimeException e) { 1797 // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage 1798 // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the 1799 // bug went unchecked for a few years and now its behavior cannot be changed 1800 // without breaking backwards compatibility. 1801 1802 if (!e.getClass().equals(RuntimeException.class)) { 1803 // re-throw any exceptions that aren't base RuntimeException since they are 1804 // coming from elsewhere, and we shouldn't silently drop those. 1805 throw e; 1806 } 1807 1808 Log.e(TAG, "Failed to detach image!"); 1809 img.close(); 1810 return; 1811 } 1812 1813 long timestamp = img.getTimestamp(); 1814 int idx = mPendingResultMap.indexOfKey(timestamp); 1815 if (idx >= 0) { 1816 boolean processStatus = true; 1817 ParcelImage parcelImage = initializeParcelImage(img); 1818 try { 1819 mPreviewImageProcessor.process(parcelImage, 1820 mPendingResultMap.get(timestamp).second, mCaptureResultHandler); 1821 } catch (RemoteException e) { 1822 processStatus = false; 1823 Log.e(TAG, "Extension service does not respond during " + 1824 "processing, dropping frame!"); 1825 } finally { 1826 parcelImage.buffer.close(); 1827 img.close(); 1828 } 1829 discardPendingRepeatingResults(idx, mPendingResultMap, false); 1830 if (mClientNotificationsEnabled) { 1831 final long ident = Binder.clearCallingIdentity(); 1832 try { 1833 if (processStatus) { 1834 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( 1835 CameraExtensionSessionImpl.this, 1836 mClientRequest)); 1837 } else { 1838 mExecutor.execute(() -> mCallbacks.onCaptureFailed( 1839 CameraExtensionSessionImpl.this, 1840 mClientRequest)); 1841 } 1842 } finally { 1843 Binder.restoreCallingIdentity(ident); 1844 } 1845 } 1846 } else { 1847 mPendingResultMap.put(timestamp, new Pair<>(img, null)); 1848 } 1849 } 1850 } 1851 } 1852 initializeFilteredResults(TotalCaptureResult result)1853 private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) { 1854 CameraMetadataNative captureResults = new CameraMetadataNative(); 1855 for (CaptureResult.Key key : mSupportedResultKeys) { 1856 Object value = result.get(key); 1857 if (value != null) { 1858 captureResults.set(key, value); 1859 } 1860 } 1861 return captureResults; 1862 } 1863 findSmallestAspectMatchedSize(@onNull List<Size> sizes, @NonNull Size arSize)1864 private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes, 1865 @NonNull Size arSize) { 1866 final float TOLL = .01f; 1867 1868 if (arSize.getHeight() == 0) { 1869 throw new IllegalArgumentException("Invalid input aspect ratio"); 1870 } 1871 1872 float targetAR = ((float) arSize.getWidth()) / arSize.getHeight(); 1873 Size ret = null; 1874 Size fallbackSize = null; 1875 for (Size sz : sizes) { 1876 if (fallbackSize == null) { 1877 fallbackSize = sz; 1878 } 1879 if ((sz.getHeight() > 0) && 1880 ((ret == null) || 1881 (ret.getWidth() * ret.getHeight()) < 1882 (sz.getWidth() * sz.getHeight()))) { 1883 float currentAR = ((float) sz.getWidth()) / sz.getHeight(); 1884 if (Math.abs(currentAR - targetAR) <= TOLL) { 1885 ret = sz; 1886 } 1887 } 1888 } 1889 if (ret == null) { 1890 Log.e(TAG, "AR matched size not found returning first size in list"); 1891 ret = fallbackSize; 1892 } 1893 1894 return ret; 1895 } 1896 initializeParcelImage(Image img)1897 private static ParcelImage initializeParcelImage(Image img) { 1898 ParcelImage parcelImage = new ParcelImage(); 1899 parcelImage.buffer = img.getHardwareBuffer(); 1900 try { 1901 SyncFence fd = img.getFence(); 1902 if (fd.isValid()) { 1903 parcelImage.fence = fd.getFdDup(); 1904 } 1905 } catch (IOException e) { 1906 Log.e(TAG, "Failed to parcel buffer fence!"); 1907 } 1908 parcelImage.width = img.getWidth(); 1909 parcelImage.height = img.getHeight(); 1910 parcelImage.format = img.getFormat(); 1911 parcelImage.timestamp = img.getTimestamp(); 1912 parcelImage.transform = img.getTransform(); 1913 parcelImage.scalingMode = img.getScalingMode(); 1914 parcelImage.planeCount = img.getPlaneCount(); 1915 parcelImage.crop = img.getCropRect(); 1916 1917 return parcelImage; 1918 } 1919 initializeParcelable( HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, Byte jpegQuality)1920 private static List<CaptureBundle> initializeParcelable( 1921 HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, 1922 Byte jpegQuality) { 1923 ArrayList<CaptureBundle> ret = new ArrayList<>(); 1924 for (Integer stagetId : captureMap.keySet()) { 1925 Pair<Image, TotalCaptureResult> entry = captureMap.get(stagetId); 1926 CaptureBundle bundle = new CaptureBundle(); 1927 bundle.stage = stagetId; 1928 bundle.captureImage = initializeParcelImage(entry.first); 1929 bundle.sequenceId = entry.second.getSequenceId(); 1930 bundle.captureResult = entry.second.getNativeMetadata(); 1931 if (jpegOrientation != null) { 1932 bundle.captureResult.set(CaptureResult.JPEG_ORIENTATION, jpegOrientation); 1933 } 1934 if (jpegQuality != null) { 1935 bundle.captureResult.set(CaptureResult.JPEG_QUALITY, jpegQuality); 1936 } 1937 ret.add(bundle); 1938 } 1939 1940 return ret; 1941 } 1942 } 1943