1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.RequiresPermission; 22 import android.content.Context; 23 import android.graphics.ImageFormat; 24 import android.graphics.SurfaceTexture; 25 import android.hardware.HardwareBuffer; 26 import android.hardware.SyncFence; 27 import android.hardware.camera2.CameraAccessException; 28 import android.hardware.camera2.CameraCaptureSession; 29 import android.hardware.camera2.CameraCharacteristics; 30 import android.hardware.camera2.CameraDevice; 31 import android.hardware.camera2.CameraExtensionCharacteristics; 32 import android.hardware.camera2.CameraExtensionSession; 33 import android.hardware.camera2.CaptureFailure; 34 import android.hardware.camera2.CaptureRequest; 35 import android.hardware.camera2.CaptureResult; 36 import android.hardware.camera2.TotalCaptureResult; 37 import android.hardware.camera2.extension.CaptureBundle; 38 import android.hardware.camera2.extension.CaptureStageImpl; 39 import android.hardware.camera2.extension.ICaptureProcessorImpl; 40 import android.hardware.camera2.extension.IImageCaptureExtenderImpl; 41 import android.hardware.camera2.extension.IInitializeSessionCallback; 42 import android.hardware.camera2.extension.IPreviewExtenderImpl; 43 import android.hardware.camera2.extension.IProcessResultImpl; 44 import android.hardware.camera2.extension.IRequestUpdateProcessorImpl; 45 import android.hardware.camera2.extension.LatencyPair; 46 import android.hardware.camera2.extension.ParcelImage; 47 import android.hardware.camera2.params.DynamicRangeProfiles; 48 import android.hardware.camera2.params.ExtensionSessionConfiguration; 49 import android.hardware.camera2.params.OutputConfiguration; 50 import android.hardware.camera2.params.SessionConfiguration; 51 import android.hardware.camera2.utils.ExtensionSessionStatsAggregator; 52 import android.hardware.camera2.utils.SurfaceUtils; 53 import android.media.Image; 54 import android.media.ImageReader; 55 import android.media.ImageWriter; 56 import android.os.Binder; 57 import android.os.Handler; 58 import android.os.HandlerThread; 59 import android.os.IBinder; 60 import android.os.RemoteException; 61 import android.util.IntArray; 62 import android.util.Log; 63 import android.util.LongSparseArray; 64 import android.util.Pair; 65 import android.util.Size; 66 import android.view.Surface; 67 68 import com.android.internal.camera.flags.Flags; 69 70 import java.io.Closeable; 71 import java.io.IOException; 72 import java.util.ArrayList; 73 import java.util.Arrays; 74 import java.util.HashMap; 75 import java.util.List; 76 import java.util.Map; 77 import java.util.Set; 78 import java.util.concurrent.Executor; 79 80 public final class CameraExtensionSessionImpl extends CameraExtensionSession { 81 private static final int PREVIEW_QUEUE_SIZE = 10; 82 private static final String TAG = "CameraExtensionSessionImpl"; 83 84 private final Executor mExecutor; 85 private final CameraDevice mCameraDevice; 86 private final IImageCaptureExtenderImpl mImageExtender; 87 private final IPreviewExtenderImpl mPreviewExtender; 88 private final Handler mHandler; 89 private final HandlerThread mHandlerThread; 90 private final StateCallback mCallbacks; 91 private final List<Size> mSupportedPreviewSizes; 92 private final InitializeSessionHandler mInitializeHandler; 93 private final int mSessionId; 94 private final Set<CaptureRequest.Key> mSupportedRequestKeys; 95 private final Set<CaptureResult.Key> mSupportedResultKeys; 96 private final ExtensionSessionStatsAggregator mStatsAggregator; 97 private IBinder mToken = null; 98 private boolean mCaptureResultsSupported; 99 100 private CameraCaptureSession mCaptureSession = null; 101 private Surface mCameraRepeatingSurface, mClientRepeatingRequestSurface; 102 private Surface mCameraBurstSurface, mClientCaptureSurface; 103 private Surface mClientPostviewSurface; 104 private ImageReader mRepeatingRequestImageReader = null; 105 private ImageReader mBurstCaptureImageReader = null; 106 private ImageReader mStubCaptureImageReader = null; 107 private ImageWriter mRepeatingRequestImageWriter = null; 108 private CameraOutputImageCallback mRepeatingRequestImageCallback = null; 109 private CameraOutputImageCallback mBurstCaptureImageCallback = null; 110 111 private CameraExtensionJpegProcessor mImageJpegProcessor = null; 112 private ICaptureProcessorImpl mImageProcessor = null; 113 private CameraExtensionForwardProcessor mPreviewImageProcessor = null; 114 private IRequestUpdateProcessorImpl mPreviewRequestUpdateProcessor = null; 115 private int mPreviewProcessorType = IPreviewExtenderImpl.PROCESSOR_TYPE_NONE; 116 117 private boolean mInitialized; 118 private boolean mSessionClosed; 119 // Enable/Disable internal preview/(repeating request). Extensions expect 120 // that preview/(repeating request) is enabled and active at any point in time. 121 // In case the client doesn't explicitly enable repeating requests, the framework 122 // will do so internally. 123 private boolean mInternalRepeatingRequestEnabled = true; 124 private int mExtensionType; 125 126 private final Context mContext; 127 128 // Lock to synchronize cross-thread access to device public interface 129 final Object mInterfaceLock; 130 nativeGetSurfaceFormat(Surface surface)131 private static int nativeGetSurfaceFormat(Surface surface) { 132 return SurfaceUtils.getSurfaceFormat(surface); 133 } 134 135 /** 136 * @hide 137 */ 138 @RequiresPermission(android.Manifest.permission.CAMERA) createCameraExtensionSession( @onNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, @NonNull Map<String, CameraCharacteristics> characteristicsMap, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId, @NonNull IBinder token)139 public static CameraExtensionSessionImpl createCameraExtensionSession( 140 @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, 141 @NonNull Map<String, CameraCharacteristics> characteristicsMap, 142 @NonNull Context ctx, 143 @NonNull ExtensionSessionConfiguration config, 144 int sessionId, 145 @NonNull IBinder token) 146 throws CameraAccessException, RemoteException { 147 String cameraId = cameraDevice.getId(); 148 CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, 149 cameraId, characteristicsMap); 150 151 if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), 152 config.getExtension(), 153 CameraExtensionUtils.getCharacteristicsMapNative(characteristicsMap))) { 154 throw new UnsupportedOperationException("Unsupported extension type: " + 155 config.getExtension()); 156 } 157 158 if (config.getOutputConfigurations().isEmpty() || 159 config.getOutputConfigurations().size() > 2) { 160 throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + 161 config.getOutputConfigurations().size() + " expected <= 2"); 162 } 163 164 for (OutputConfiguration c : config.getOutputConfigurations()) { 165 if (c.getDynamicRangeProfile() != DynamicRangeProfiles.STANDARD) { 166 throw new IllegalArgumentException("Unsupported dynamic range profile: " + 167 c.getDynamicRangeProfile()); 168 } 169 if (c.getStreamUseCase() != 170 CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT) { 171 throw new IllegalArgumentException("Unsupported stream use case: " + 172 c.getStreamUseCase()); 173 } 174 } 175 176 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 177 CameraExtensionCharacteristics.initializeExtension(config.getExtension()); 178 179 int suitableSurfaceCount = 0; 180 List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( 181 config.getExtension(), SurfaceTexture.class); 182 Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface( 183 config.getOutputConfigurations(), supportedPreviewSizes); 184 if (repeatingRequestSurface != null) { 185 suitableSurfaceCount++; 186 } 187 188 HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>(); 189 Integer[] supportedCaptureOutputFormats = 190 new Integer[CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS.size()]; 191 supportedCaptureOutputFormats = 192 CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS.toArray( 193 supportedCaptureOutputFormats); 194 for (int format : supportedCaptureOutputFormats) { 195 List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes( 196 config.getExtension(), format); 197 if (supportedSizes != null) { 198 supportedCaptureSizes.put(format, supportedSizes); 199 } 200 } 201 202 int captureFormat = ImageFormat.UNKNOWN; 203 Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface( 204 config.getOutputConfigurations(), supportedCaptureSizes); 205 if (burstCaptureSurface != null) { 206 suitableSurfaceCount++; 207 208 if (Flags.analytics24q3()) { 209 CameraExtensionUtils.SurfaceInfo burstCaptureSurfaceInfo = 210 CameraExtensionUtils.querySurface(burstCaptureSurface); 211 captureFormat = burstCaptureSurfaceInfo.mFormat; 212 } 213 } 214 215 if (suitableSurfaceCount != config.getOutputConfigurations().size()) { 216 throw new IllegalArgumentException("One or more unsupported output surfaces found!"); 217 } 218 219 Surface postviewSurface = null; 220 if (burstCaptureSurface != null && config.getPostviewOutputConfiguration() != null) { 221 CameraExtensionUtils.SurfaceInfo burstCaptureSurfaceInfo = 222 CameraExtensionUtils.querySurface(burstCaptureSurface); 223 Size burstCaptureSurfaceSize = 224 new Size(burstCaptureSurfaceInfo.mWidth, burstCaptureSurfaceInfo.mHeight); 225 HashMap<Integer, List<Size>> supportedPostviewSizes = new HashMap<>(); 226 for (int format : supportedCaptureOutputFormats) { 227 List<Size> supportedSizesPostview = extensionChars.getPostviewSupportedSizes( 228 config.getExtension(), burstCaptureSurfaceSize, format); 229 if (supportedSizesPostview != null) { 230 supportedPostviewSizes.put(format, supportedSizesPostview); 231 } 232 } 233 234 postviewSurface = CameraExtensionUtils.getPostviewSurface( 235 config.getPostviewOutputConfiguration(), supportedPostviewSizes, 236 burstCaptureSurfaceInfo.mFormat); 237 238 if (postviewSurface == null) { 239 throw new IllegalArgumentException("Unsupported output surface for postview!"); 240 } 241 } 242 243 extenders.first.init(cameraId, characteristicsMap.get(cameraId).getNativeMetadata()); 244 extenders.first.onInit(token, cameraId, 245 characteristicsMap.get(cameraId).getNativeMetadata()); 246 extenders.second.init(cameraId, characteristicsMap.get(cameraId).getNativeMetadata()); 247 extenders.second.onInit(token, cameraId, 248 characteristicsMap.get(cameraId).getNativeMetadata()); 249 250 CameraExtensionSessionImpl session = new CameraExtensionSessionImpl( 251 ctx, 252 extenders.second, 253 extenders.first, 254 supportedPreviewSizes, 255 cameraDevice, 256 repeatingRequestSurface, 257 burstCaptureSurface, 258 postviewSurface, 259 config.getStateCallback(), 260 config.getExecutor(), 261 sessionId, 262 token, 263 extensionChars.getAvailableCaptureRequestKeys(config.getExtension()), 264 extensionChars.getAvailableCaptureResultKeys(config.getExtension()), 265 config.getExtension()); 266 267 if (Flags.analytics24q3()) { 268 session.mStatsAggregator.setCaptureFormat(captureFormat); 269 } 270 session.mStatsAggregator.setClientName(ctx.getOpPackageName()); 271 session.mStatsAggregator.setExtensionType(config.getExtension()); 272 273 session.initialize(); 274 275 return session; 276 } 277 CameraExtensionSessionImpl(Context ctx, @NonNull IImageCaptureExtenderImpl imageExtender, @NonNull IPreviewExtenderImpl previewExtender, @NonNull List<Size> previewSizes, @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @Nullable Surface postviewSurface, @NonNull StateCallback callback, @NonNull Executor executor, int sessionId, @NonNull IBinder token, @NonNull Set<CaptureRequest.Key> requestKeys, @Nullable Set<CaptureResult.Key> resultKeys, int extension)278 public CameraExtensionSessionImpl(Context ctx, @NonNull IImageCaptureExtenderImpl imageExtender, 279 @NonNull IPreviewExtenderImpl previewExtender, 280 @NonNull List<Size> previewSizes, 281 @NonNull android.hardware.camera2.impl.CameraDeviceImpl cameraDevice, 282 @Nullable Surface repeatingRequestSurface, 283 @Nullable Surface burstCaptureSurface, 284 @Nullable Surface postviewSurface, 285 @NonNull StateCallback callback, 286 @NonNull Executor executor, 287 int sessionId, 288 @NonNull IBinder token, 289 @NonNull Set<CaptureRequest.Key> requestKeys, 290 @Nullable Set<CaptureResult.Key> resultKeys, 291 int extension) { 292 mContext = ctx; 293 mImageExtender = imageExtender; 294 mPreviewExtender = previewExtender; 295 mCameraDevice = cameraDevice; 296 mCallbacks = callback; 297 mExecutor = executor; 298 mClientRepeatingRequestSurface = repeatingRequestSurface; 299 mClientCaptureSurface = burstCaptureSurface; 300 mClientPostviewSurface = postviewSurface; 301 mSupportedPreviewSizes = previewSizes; 302 mHandlerThread = new HandlerThread(TAG); 303 mHandlerThread.start(); 304 mHandler = new Handler(mHandlerThread.getLooper()); 305 mInitialized = false; 306 mSessionClosed = false; 307 mInitializeHandler = new InitializeSessionHandler(); 308 mSessionId = sessionId; 309 mToken = token; 310 mSupportedRequestKeys = requestKeys; 311 mSupportedResultKeys = resultKeys; 312 mCaptureResultsSupported = !resultKeys.isEmpty(); 313 mInterfaceLock = cameraDevice.mInterfaceLock; 314 mExtensionType = extension; 315 316 mStatsAggregator = new ExtensionSessionStatsAggregator(mCameraDevice.getId(), 317 /*isAdvanced=*/false); 318 } 319 initializeRepeatingRequestPipeline()320 private void initializeRepeatingRequestPipeline() throws RemoteException { 321 CameraExtensionUtils.SurfaceInfo repeatingSurfaceInfo = 322 new CameraExtensionUtils.SurfaceInfo(); 323 mPreviewProcessorType = mPreviewExtender.getProcessorType(); 324 if (mClientRepeatingRequestSurface != null) { 325 repeatingSurfaceInfo = CameraExtensionUtils.querySurface( 326 mClientRepeatingRequestSurface); 327 } else { 328 // Make the intermediate surface behave as any regular 'SurfaceTexture' 329 CameraExtensionUtils.SurfaceInfo captureSurfaceInfo = CameraExtensionUtils.querySurface( 330 mClientCaptureSurface); 331 Size captureSize = new Size(captureSurfaceInfo.mWidth, captureSurfaceInfo.mHeight); 332 Size previewSize = findSmallestAspectMatchedSize(mSupportedPreviewSizes, captureSize); 333 repeatingSurfaceInfo.mWidth = previewSize.getWidth(); 334 repeatingSurfaceInfo.mHeight = previewSize.getHeight(); 335 repeatingSurfaceInfo.mUsage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE; 336 } 337 338 if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { 339 try { 340 mPreviewImageProcessor = new CameraExtensionForwardProcessor( 341 mPreviewExtender.getPreviewImageProcessor(), repeatingSurfaceInfo.mFormat, 342 repeatingSurfaceInfo.mUsage, mHandler); 343 } catch (ClassCastException e) { 344 throw new UnsupportedOperationException("Failed casting preview processor!"); 345 } 346 mPreviewImageProcessor.onImageFormatUpdate( 347 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); 348 mPreviewImageProcessor.onResolutionUpdate(new Size(repeatingSurfaceInfo.mWidth, 349 repeatingSurfaceInfo.mHeight)); 350 mPreviewImageProcessor.onOutputSurface(null, -1); 351 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 352 repeatingSurfaceInfo.mHeight, 353 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, PREVIEW_QUEUE_SIZE, 354 repeatingSurfaceInfo.mUsage); 355 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 356 } else if (mPreviewProcessorType == 357 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { 358 try { 359 mPreviewRequestUpdateProcessor = mPreviewExtender.getRequestUpdateProcessor(); 360 } catch (ClassCastException e) { 361 throw new UnsupportedOperationException("Failed casting preview processor!"); 362 } 363 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 364 repeatingSurfaceInfo.mHeight, 365 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, 366 PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); 367 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 368 android.hardware.camera2.extension.Size sz = 369 new android.hardware.camera2.extension.Size(); 370 sz.width = repeatingSurfaceInfo.mWidth; 371 sz.height = repeatingSurfaceInfo.mHeight; 372 mPreviewRequestUpdateProcessor.onResolutionUpdate(sz); 373 mPreviewRequestUpdateProcessor.onImageFormatUpdate( 374 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 375 } else { 376 mRepeatingRequestImageReader = ImageReader.newInstance(repeatingSurfaceInfo.mWidth, 377 repeatingSurfaceInfo.mHeight, 378 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT, 379 PREVIEW_QUEUE_SIZE, repeatingSurfaceInfo.mUsage); 380 mCameraRepeatingSurface = mRepeatingRequestImageReader.getSurface(); 381 } 382 mRepeatingRequestImageCallback = new CameraOutputImageCallback( 383 mRepeatingRequestImageReader, true /*pruneOlderBuffers*/); 384 mRepeatingRequestImageReader 385 .setOnImageAvailableListener(mRepeatingRequestImageCallback, mHandler); 386 } 387 initializeBurstCapturePipeline()388 private void initializeBurstCapturePipeline() throws RemoteException { 389 mImageProcessor = mImageExtender.getCaptureProcessor(); 390 if ((mImageProcessor == null) && (mImageExtender.getMaxCaptureStage() != 1)) { 391 throw new UnsupportedOperationException("Multiple stages expected without" + 392 " a valid capture processor!"); 393 } 394 395 if (mImageProcessor != null) { 396 if (mClientCaptureSurface != null) { 397 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface( 398 mClientCaptureSurface); 399 if (surfaceInfo.mFormat == ImageFormat.JPEG) { 400 mImageJpegProcessor = new CameraExtensionJpegProcessor(mImageProcessor); 401 mImageProcessor = mImageJpegProcessor; 402 } else if (mClientPostviewSurface != null) { 403 // Handles case when postview is JPEG and capture is YUV 404 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo = 405 CameraExtensionUtils.querySurface(mClientPostviewSurface); 406 if (postviewSurfaceInfo.mFormat == ImageFormat.JPEG) { 407 mImageJpegProcessor = new CameraExtensionJpegProcessor(mImageProcessor); 408 mImageProcessor = mImageJpegProcessor; 409 } 410 } 411 412 mBurstCaptureImageReader = ImageReader.newInstance(surfaceInfo.mWidth, 413 surfaceInfo.mHeight, CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 414 mImageExtender.getMaxCaptureStage()); 415 } else { 416 // The client doesn't intend to trigger multi-frame capture, however the 417 // image extender still needs to get initialized and the camera still capture 418 // stream configured for the repeating request processing to work. 419 mBurstCaptureImageReader = ImageReader.newInstance( 420 mRepeatingRequestImageReader.getWidth(), 421 mRepeatingRequestImageReader.getHeight(), 422 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); 423 // The still capture output is not going to be used but we still need a valid 424 // surface to register. 425 mStubCaptureImageReader = ImageReader.newInstance( 426 mRepeatingRequestImageReader.getWidth(), 427 mRepeatingRequestImageReader.getHeight(), 428 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT, 1); 429 mImageProcessor.onOutputSurface(mStubCaptureImageReader.getSurface(), 430 CameraExtensionCharacteristics.PROCESSING_INPUT_FORMAT); 431 } 432 433 mBurstCaptureImageCallback = new CameraOutputImageCallback(mBurstCaptureImageReader, 434 false /*pruneOlderBuffers*/); 435 mBurstCaptureImageReader.setOnImageAvailableListener(mBurstCaptureImageCallback, 436 mHandler); 437 mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); 438 android.hardware.camera2.extension.Size sz = 439 new android.hardware.camera2.extension.Size(); 440 sz.width = mBurstCaptureImageReader.getWidth(); 441 sz.height = mBurstCaptureImageReader.getHeight(); 442 443 if (mClientPostviewSurface != null) { 444 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo = 445 CameraExtensionUtils.querySurface(mClientPostviewSurface); 446 android.hardware.camera2.extension.Size postviewSize = 447 new android.hardware.camera2.extension.Size(); 448 postviewSize.width = postviewSurfaceInfo.mWidth; 449 postviewSize.height = postviewSurfaceInfo.mHeight; 450 mImageProcessor.onResolutionUpdate(sz, postviewSize); 451 } else { 452 mImageProcessor.onResolutionUpdate(sz, null); 453 } 454 455 mImageProcessor.onImageFormatUpdate(mBurstCaptureImageReader.getImageFormat()); 456 } else { 457 if (mClientCaptureSurface != null) { 458 // Redirect camera output directly in to client output surface 459 mCameraBurstSurface = mClientCaptureSurface; 460 } else { 461 // The client doesn't intend to trigger multi-frame capture, however the 462 // image extender still needs to get initialized and the camera still capture 463 // stream configured for the repeating request processing to work. 464 mBurstCaptureImageReader = ImageReader.newInstance( 465 mRepeatingRequestImageReader.getWidth(), 466 mRepeatingRequestImageReader.getHeight(), 467 // Camera devices accept only Jpeg output if the image processor is null 468 ImageFormat.JPEG, 1); 469 mCameraBurstSurface = mBurstCaptureImageReader.getSurface(); 470 } 471 } 472 } 473 finishPipelineInitialization()474 private void finishPipelineInitialization() throws RemoteException { 475 if (mClientRepeatingRequestSurface != null) { 476 if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) { 477 mPreviewRequestUpdateProcessor.onOutputSurface(mClientRepeatingRequestSurface, 478 nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); 479 mRepeatingRequestImageWriter = ImageWriter.newInstance( 480 mClientRepeatingRequestSurface, 481 PREVIEW_QUEUE_SIZE, 482 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 483 } else if (mPreviewProcessorType == IPreviewExtenderImpl.PROCESSOR_TYPE_NONE) { 484 mRepeatingRequestImageWriter = ImageWriter.newInstance( 485 mClientRepeatingRequestSurface, 486 PREVIEW_QUEUE_SIZE, 487 CameraExtensionCharacteristics.NON_PROCESSING_INPUT_FORMAT); 488 } 489 } 490 if ((mImageProcessor != null) && (mClientCaptureSurface != null)) { 491 if (mClientPostviewSurface != null) { 492 mImageProcessor.onPostviewOutputSurface(mClientPostviewSurface); 493 } 494 495 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface( 496 mClientCaptureSurface); 497 mImageProcessor.onOutputSurface(mClientCaptureSurface, surfaceInfo.mFormat); 498 } 499 } 500 501 /** 502 * @hide 503 */ initialize()504 public synchronized void initialize() throws CameraAccessException, RemoteException { 505 if (mInitialized) { 506 Log.d(TAG, 507 "Session already initialized"); 508 return; 509 } 510 int previewSessionType = mPreviewExtender.getSessionType(); 511 int imageSessionType = mImageExtender.getSessionType(); 512 if (previewSessionType != imageSessionType) { 513 throw new IllegalStateException("Preview extender session type: " + previewSessionType + 514 "and image extender session type: " + imageSessionType + " mismatch!"); 515 } 516 int sessionType = SessionConfiguration.SESSION_REGULAR; 517 if ((previewSessionType != -1) && 518 (previewSessionType != SessionConfiguration.SESSION_HIGH_SPEED)) { 519 sessionType = previewSessionType; 520 Log.v(TAG, "Using session type: " + sessionType); 521 } 522 523 ArrayList<CaptureStageImpl> sessionParamsList = new ArrayList<>(); 524 ArrayList<OutputConfiguration> outputList = new ArrayList<>(); 525 initializeRepeatingRequestPipeline(); 526 OutputConfiguration previewOutput = new OutputConfiguration(mCameraRepeatingSurface); 527 // The extension processing logic needs to be able to match images to capture results via 528 // image and result timestamps. 529 previewOutput.setTimestampBase(OutputConfiguration.TIMESTAMP_BASE_SENSOR); 530 previewOutput.setReadoutTimestampEnabled(false); 531 outputList.add(previewOutput); 532 CaptureStageImpl previewSessionParams = mPreviewExtender.onPresetSession(); 533 if (previewSessionParams != null) { 534 sessionParamsList.add(previewSessionParams); 535 } 536 initializeBurstCapturePipeline(); 537 OutputConfiguration captureOutput = new OutputConfiguration(mCameraBurstSurface); 538 captureOutput.setTimestampBase(OutputConfiguration.TIMESTAMP_BASE_SENSOR); 539 captureOutput.setReadoutTimestampEnabled(false); 540 outputList.add(captureOutput); 541 CaptureStageImpl stillCaptureSessionParams = mImageExtender.onPresetSession(); 542 if (stillCaptureSessionParams != null) { 543 sessionParamsList.add(stillCaptureSessionParams); 544 } 545 546 SessionConfiguration sessionConfig = new SessionConfiguration( 547 sessionType, 548 outputList, 549 new CameraExtensionUtils.HandlerExecutor(mHandler), 550 new SessionStateHandler()); 551 552 if (!sessionParamsList.isEmpty()) { 553 CaptureRequest sessionParamRequest = createRequest(mCameraDevice, sessionParamsList, 554 null, CameraDevice.TEMPLATE_PREVIEW); 555 sessionConfig.setSessionParameters(sessionParamRequest); 556 } 557 558 mCameraDevice.createCaptureSession(sessionConfig); 559 } 560 561 @Override getDevice()562 public @NonNull CameraDevice getDevice() { 563 synchronized (mInterfaceLock) { 564 return mCameraDevice; 565 } 566 } 567 568 @Override getRealtimeStillCaptureLatency()569 public StillCaptureLatency getRealtimeStillCaptureLatency() throws CameraAccessException { 570 synchronized (mInterfaceLock) { 571 if (!mInitialized) { 572 throw new IllegalStateException("Uninitialized component"); 573 } 574 575 try { 576 LatencyPair latency = mImageExtender.getRealtimeCaptureLatency(); 577 if (latency != null) { 578 return new StillCaptureLatency(latency.first, latency.second); 579 } 580 581 return null; 582 } catch (RemoteException e) { 583 Log.e(TAG, "Failed to query realtime latency! Extension service does not " 584 + "respond"); 585 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 586 } 587 } 588 } 589 590 @Override setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)591 public int setRepeatingRequest(@NonNull CaptureRequest request, 592 @NonNull Executor executor, 593 @NonNull ExtensionCaptureCallback listener) 594 throws CameraAccessException { 595 synchronized (mInterfaceLock) { 596 if (!mInitialized) { 597 throw new IllegalStateException("Uninitialized component"); 598 } 599 600 if (mClientRepeatingRequestSurface == null) { 601 throw new IllegalArgumentException("No registered preview surface"); 602 } 603 604 if (!request.containsTarget(mClientRepeatingRequestSurface) || 605 (request.getTargets().size() != 1)) { 606 throw new IllegalArgumentException("Invalid repeating request output target!"); 607 } 608 609 mInternalRepeatingRequestEnabled = false; 610 try { 611 return setRepeatingRequest(mPreviewExtender.getCaptureStage(), 612 new PreviewRequestHandler(request, executor, listener, 613 mRepeatingRequestImageCallback), request); 614 } catch (RemoteException e) { 615 Log.e(TAG, "Failed to set repeating request! Extension service does not " 616 + "respond"); 617 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 618 } 619 } 620 } 621 compileInitialRequestList()622 private ArrayList<CaptureStageImpl> compileInitialRequestList() { 623 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 624 try { 625 CaptureStageImpl initialPreviewParams = mPreviewExtender.onEnableSession(); 626 if (initialPreviewParams != null) { 627 captureStageList.add(initialPreviewParams); 628 } 629 630 CaptureStageImpl initialStillCaptureParams = mImageExtender.onEnableSession(); 631 if (initialStillCaptureParams != null) { 632 captureStageList.add(initialStillCaptureParams); 633 } 634 } catch (RemoteException e) { 635 Log.e(TAG, "Failed to initialize session parameters! Extension service does not" 636 + " respond!"); 637 } 638 639 return captureStageList; 640 } 641 createBurstRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap)642 private List<CaptureRequest> createBurstRequest(CameraDevice cameraDevice, 643 List<CaptureStageImpl> captureStageList, CaptureRequest clientRequest, 644 Surface target, int captureTemplate, Map<CaptureRequest, Integer> captureMap) { 645 CaptureRequest.Builder requestBuilder; 646 ArrayList<CaptureRequest> ret = new ArrayList<>(); 647 for (CaptureStageImpl captureStage : captureStageList) { 648 try { 649 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); 650 } catch (CameraAccessException e) { 651 return null; 652 } 653 654 // This will guarantee that client configured 655 // parameters always have the highest priority. 656 for (CaptureRequest.Key requestKey : mSupportedRequestKeys){ 657 Object value = clientRequest.get(requestKey); 658 if (value != null) { 659 captureStage.parameters.set(requestKey, value); 660 } 661 } 662 663 requestBuilder.addTarget(target); 664 CaptureRequest request = requestBuilder.build(); 665 CameraMetadataNative.update(request.getNativeMetadata(), captureStage.parameters); 666 ret.add(request); 667 if (captureMap != null) { 668 captureMap.put(request, captureStage.id); 669 } 670 } 671 672 return ret; 673 } 674 createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate, CaptureRequest clientRequest)675 private CaptureRequest createRequest(CameraDevice cameraDevice, 676 List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate, 677 CaptureRequest clientRequest) throws CameraAccessException { 678 CaptureRequest.Builder requestBuilder; 679 requestBuilder = cameraDevice.createCaptureRequest(captureTemplate); 680 if (target != null) { 681 requestBuilder.addTarget(target); 682 } 683 684 CaptureRequest ret = requestBuilder.build(); 685 CameraMetadataNative nativeMeta = ret.getNativeMetadata(); 686 for (CaptureStageImpl captureStage : captureStageList) { 687 if (captureStage != null) { 688 CameraMetadataNative.update(nativeMeta, captureStage.parameters); 689 } 690 } 691 692 if (clientRequest != null) { 693 // This will guarantee that client configured 694 // parameters always have the highest priority. 695 for (CaptureRequest.Key requestKey : mSupportedRequestKeys) { 696 Object value = clientRequest.get(requestKey); 697 if (value != null) { 698 nativeMeta.set(requestKey, value); 699 } 700 } 701 } 702 703 return ret; 704 } 705 createRequest(CameraDevice cameraDevice, List<CaptureStageImpl> captureStageList, Surface target, int captureTemplate)706 private CaptureRequest createRequest(CameraDevice cameraDevice, 707 List<CaptureStageImpl> captureStageList, 708 Surface target, 709 int captureTemplate) throws CameraAccessException { 710 return createRequest(cameraDevice, captureStageList, target, captureTemplate, 711 /*clientRequest*/ null); 712 } 713 714 @Override capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)715 public int capture(@NonNull CaptureRequest request, 716 @NonNull Executor executor, 717 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 718 if (!mInitialized) { 719 throw new IllegalStateException("Uninitialized component"); 720 } 721 722 validateCaptureRequestTargets(request); 723 724 int seqId = -1; 725 if ((mClientCaptureSurface != null) && request.containsTarget(mClientCaptureSurface)) { 726 HashMap<CaptureRequest, Integer> requestMap = new HashMap<>(); 727 List<CaptureRequest> burstRequest; 728 try { 729 burstRequest = createBurstRequest(mCameraDevice, 730 mImageExtender.getCaptureStages(), request, mCameraBurstSurface, 731 CameraDevice.TEMPLATE_STILL_CAPTURE, requestMap); 732 } catch (RemoteException e) { 733 Log.e(TAG, "Failed to initialize internal burst request! Extension service does" 734 + " not respond!"); 735 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 736 } 737 if (burstRequest == null) { 738 throw new UnsupportedOperationException( 739 "Failed to create still capture burst request"); 740 } 741 742 seqId = mCaptureSession.captureBurstRequests(burstRequest, 743 new CameraExtensionUtils.HandlerExecutor(mHandler), 744 new BurstRequestHandler(request, executor, listener, requestMap, 745 mBurstCaptureImageCallback)); 746 } else if ((mClientRepeatingRequestSurface != null) && 747 request.containsTarget(mClientRepeatingRequestSurface)) { 748 749 CaptureRequest captureRequest = null; 750 try { 751 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 752 captureStageList.add(mPreviewExtender.getCaptureStage()); 753 754 captureRequest = createRequest(mCameraDevice, captureStageList, 755 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, request); 756 } catch (RemoteException e) { 757 Log.e(TAG, "Failed to initialize capture request! Extension service does" 758 + " not respond!"); 759 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR); 760 } 761 762 seqId = mCaptureSession.capture(captureRequest, new PreviewRequestHandler(request, 763 executor, listener, mRepeatingRequestImageCallback, true /*singleCapture*/), 764 mHandler); 765 } else { 766 throw new IllegalArgumentException("Capture request to unknown output surface!"); 767 } 768 769 return seqId; 770 } 771 validateCaptureRequestTargets(@onNull CaptureRequest request)772 private void validateCaptureRequestTargets(@NonNull CaptureRequest request) { 773 if (request.getTargets().size() == 1) { 774 boolean containsCaptureTarget = 775 mClientCaptureSurface != null && request.containsTarget(mClientCaptureSurface); 776 boolean containsRepeatingTarget = 777 mClientRepeatingRequestSurface != null && 778 request.containsTarget(mClientRepeatingRequestSurface); 779 780 if (!containsCaptureTarget && !containsRepeatingTarget) { 781 throw new IllegalArgumentException("Target output combination requested is " + 782 "not supported!"); 783 } 784 } 785 786 if ((request.getTargets().size() == 2) && 787 (!request.getTargets().containsAll(Arrays.asList(mClientCaptureSurface, 788 mClientPostviewSurface)))) { 789 throw new IllegalArgumentException("Target output combination requested is " + 790 "not supported!"); 791 } 792 793 if (request.getTargets().size() > 2) { 794 throw new IllegalArgumentException("Target output combination requested is " + 795 "not supported!"); 796 } 797 } 798 799 @Override stopRepeating()800 public void stopRepeating() throws CameraAccessException { 801 synchronized (mInterfaceLock) { 802 if (!mInitialized) { 803 throw new IllegalStateException("Uninitialized component"); 804 } 805 806 mInternalRepeatingRequestEnabled = true; 807 mCaptureSession.stopRepeating(); 808 } 809 } 810 811 @Override close()812 public void close() throws CameraAccessException { 813 synchronized (mInterfaceLock) { 814 if (mInitialized) { 815 mInternalRepeatingRequestEnabled = false; 816 try { 817 mCaptureSession.stopRepeating(); 818 } catch (IllegalStateException e) { 819 // OK: already be closed, nothing else to do 820 mSessionClosed = true; 821 } 822 823 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 824 try { 825 CaptureStageImpl disableParams = mPreviewExtender.onDisableSession(); 826 if (disableParams != null) { 827 captureStageList.add(disableParams); 828 } 829 830 CaptureStageImpl disableStillCaptureParams = 831 mImageExtender.onDisableSession(); 832 if (disableStillCaptureParams != null) { 833 captureStageList.add(disableStillCaptureParams); 834 } 835 } catch (RemoteException e) { 836 Log.e(TAG, "Failed to disable extension! Extension service does not " 837 + "respond!"); 838 } 839 if (!captureStageList.isEmpty() && !mSessionClosed) { 840 CaptureRequest disableRequest = createRequest(mCameraDevice, captureStageList, 841 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); 842 mCaptureSession.capture(disableRequest, 843 new CloseRequestHandler(mRepeatingRequestImageCallback), mHandler); 844 } 845 846 mSessionClosed = true; 847 mStatsAggregator.commit(/*isFinal*/true); // Commit stats before closing session 848 mCaptureSession.close(); 849 } 850 } 851 } 852 853 /** 854 * Called by {@link CameraDeviceImpl} right before the capture session is closed, and before it 855 * calls {@link #release} 856 * 857 * @hide 858 */ commitStats()859 public void commitStats() { 860 synchronized (mInterfaceLock) { 861 if (mInitialized) { 862 // Only commit stats if a capture session was initialized 863 mStatsAggregator.commit(/*isFinal*/true); 864 } 865 } 866 } 867 setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, InitialRequestHandler requestHandler)868 private void setInitialCaptureRequest(List<CaptureStageImpl> captureStageList, 869 InitialRequestHandler requestHandler) 870 throws CameraAccessException { 871 CaptureRequest initialRequest = createRequest(mCameraDevice, 872 captureStageList, mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW); 873 mCaptureSession.capture(initialRequest, requestHandler, mHandler); 874 } 875 setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler)876 private int setRepeatingRequest(CaptureStageImpl captureStage, 877 CameraCaptureSession.CaptureCallback requestHandler) throws CameraAccessException { 878 return setRepeatingRequest(captureStage, requestHandler, /*clientRequest*/ null); 879 } 880 setRepeatingRequest(CaptureStageImpl captureStage, CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest)881 private int setRepeatingRequest(CaptureStageImpl captureStage, 882 CameraCaptureSession.CaptureCallback requestHandler, CaptureRequest clientRequest) 883 throws CameraAccessException { 884 ArrayList<CaptureStageImpl> captureStageList = new ArrayList<>(); 885 captureStageList.add(captureStage); 886 CaptureRequest repeatingRequest = createRequest(mCameraDevice, captureStageList, 887 mCameraRepeatingSurface, CameraDevice.TEMPLATE_PREVIEW, clientRequest); 888 return mCaptureSession.setSingleRepeatingRequest(repeatingRequest, 889 new CameraExtensionUtils.HandlerExecutor(mHandler), requestHandler); 890 } 891 892 /** @hide */ release(boolean skipCloseNotification)893 public void release(boolean skipCloseNotification) { 894 boolean notifyClose = false; 895 896 synchronized (mInterfaceLock) { 897 mInternalRepeatingRequestEnabled = false; 898 mHandlerThread.quit(); 899 900 try { 901 if (!mSessionClosed) { 902 // return value is omitted. nothing can do after session is closed. 903 mPreviewExtender.onDisableSession(); 904 mImageExtender.onDisableSession(); 905 } 906 mPreviewExtender.onDeInit(mToken); 907 mImageExtender.onDeInit(mToken); 908 } catch (RemoteException e) { 909 Log.e(TAG, "Failed to release extensions! Extension service does not" 910 + " respond!"); 911 } 912 913 if (mToken != null) { 914 if (mInitialized || (mCaptureSession != null)) { 915 notifyClose = true; 916 CameraExtensionCharacteristics.releaseSession(mExtensionType); 917 } 918 CameraExtensionCharacteristics.unregisterClient(mContext, mToken, mExtensionType); 919 } 920 mInitialized = false; 921 mToken = null; 922 923 if (mRepeatingRequestImageCallback != null) { 924 mRepeatingRequestImageCallback.close(); 925 mRepeatingRequestImageCallback = null; 926 } 927 928 if (mRepeatingRequestImageReader != null) { 929 mRepeatingRequestImageReader.close(); 930 mRepeatingRequestImageReader = null; 931 } 932 933 if (mBurstCaptureImageCallback != null) { 934 mBurstCaptureImageCallback.close(); 935 mBurstCaptureImageCallback = null; 936 } 937 938 if (mBurstCaptureImageReader != null) { 939 mBurstCaptureImageReader.close(); 940 mBurstCaptureImageReader = null; 941 } 942 943 if (mStubCaptureImageReader != null) { 944 mStubCaptureImageReader.close(); 945 mStubCaptureImageReader = null; 946 } 947 948 if (mRepeatingRequestImageWriter != null) { 949 mRepeatingRequestImageWriter.close(); 950 mRepeatingRequestImageWriter = null; 951 } 952 953 if (mPreviewImageProcessor != null) { 954 mPreviewImageProcessor.close(); 955 mPreviewImageProcessor = null; 956 } 957 958 if (mImageJpegProcessor != null) { 959 mImageJpegProcessor.close(); 960 mImageJpegProcessor = null; 961 } 962 963 mCaptureSession = null; 964 mImageProcessor = null; 965 mCameraRepeatingSurface = mClientRepeatingRequestSurface = null; 966 mCameraBurstSurface = mClientCaptureSurface = null; 967 mClientPostviewSurface = null; 968 } 969 970 if (notifyClose && !skipCloseNotification) { 971 final long ident = Binder.clearCallingIdentity(); 972 try { 973 mExecutor.execute(() -> mCallbacks.onClosed(CameraExtensionSessionImpl.this)); 974 } finally { 975 Binder.restoreCallingIdentity(ident); 976 } 977 } 978 } 979 notifyConfigurationFailure()980 private void notifyConfigurationFailure() { 981 synchronized (mInterfaceLock) { 982 if (mInitialized) { 983 return; 984 } 985 } 986 987 release(true /*skipCloseNotification*/); 988 989 final long ident = Binder.clearCallingIdentity(); 990 try { 991 mExecutor.execute( 992 () -> mCallbacks.onConfigureFailed(CameraExtensionSessionImpl.this)); 993 } finally { 994 Binder.restoreCallingIdentity(ident); 995 } 996 } 997 notifyConfigurationSuccess()998 private void notifyConfigurationSuccess() { 999 synchronized (mInterfaceLock) { 1000 if (mInitialized) { 1001 return; 1002 } else { 1003 mInitialized = true; 1004 } 1005 } 1006 1007 final long ident = Binder.clearCallingIdentity(); 1008 try { 1009 mExecutor.execute(() -> mCallbacks.onConfigured(CameraExtensionSessionImpl.this)); 1010 } finally { 1011 Binder.restoreCallingIdentity(ident); 1012 } 1013 } 1014 1015 private class SessionStateHandler extends 1016 android.hardware.camera2.CameraCaptureSession.StateCallback { 1017 @Override onClosed(@onNull CameraCaptureSession session)1018 public void onClosed(@NonNull CameraCaptureSession session) { 1019 release(false /*skipCloseNotification*/); 1020 } 1021 1022 @Override onConfigureFailed(@onNull CameraCaptureSession session)1023 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 1024 notifyConfigurationFailure(); 1025 } 1026 1027 @Override onConfigured(@onNull CameraCaptureSession session)1028 public void onConfigured(@NonNull CameraCaptureSession session) { 1029 synchronized (mInterfaceLock) { 1030 mCaptureSession = session; 1031 // Commit basic stats as soon as the capture session is created 1032 mStatsAggregator.commit(/*isFinal*/false); 1033 try { 1034 finishPipelineInitialization(); 1035 CameraExtensionCharacteristics.initializeSession( 1036 mInitializeHandler, mExtensionType); 1037 } catch (RemoteException e) { 1038 Log.e(TAG, "Failed to initialize session! Extension service does" 1039 + " not respond!"); 1040 notifyConfigurationFailure(); 1041 } 1042 } 1043 } 1044 } 1045 1046 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 1047 @Override onSuccess()1048 public void onSuccess() { 1049 mHandler.post(new Runnable() { 1050 @Override 1051 public void run() { 1052 boolean status = true; 1053 ArrayList<CaptureStageImpl> initialRequestList = 1054 compileInitialRequestList(); 1055 if (!initialRequestList.isEmpty()) { 1056 try { 1057 setInitialCaptureRequest(initialRequestList, 1058 new InitialRequestHandler( 1059 mRepeatingRequestImageCallback)); 1060 } catch (CameraAccessException e) { 1061 Log.e(TAG, 1062 "Failed to initialize the initial capture " 1063 + "request!"); 1064 status = false; 1065 } 1066 } else { 1067 try { 1068 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 1069 new PreviewRequestHandler(null, null, null, 1070 mRepeatingRequestImageCallback)); 1071 } catch (CameraAccessException | RemoteException e) { 1072 Log.e(TAG, 1073 "Failed to initialize internal repeating " 1074 + "request!"); 1075 status = false; 1076 } 1077 1078 } 1079 1080 if (!status) { 1081 notifyConfigurationFailure(); 1082 } 1083 } 1084 }); 1085 } 1086 1087 @Override onFailure()1088 public void onFailure() { 1089 mHandler.post(new Runnable() { 1090 @Override 1091 public void run() { 1092 mCaptureSession.close(); 1093 Log.e(TAG, "Failed to initialize proxy service session!" 1094 + " This can happen when trying to configure multiple " 1095 + "concurrent extension sessions!"); 1096 notifyConfigurationFailure(); 1097 } 1098 }); 1099 } 1100 } 1101 1102 private class BurstRequestHandler extends CameraCaptureSession.CaptureCallback { 1103 private final Executor mExecutor; 1104 private final ExtensionCaptureCallback mCallbacks; 1105 private final CaptureRequest mClientRequest; 1106 private final HashMap<CaptureRequest, Integer> mCaptureRequestMap; 1107 private final CameraOutputImageCallback mBurstImageCallback; 1108 1109 private HashMap<Integer, Pair<Image, TotalCaptureResult>> mCaptureStageMap = 1110 new HashMap<>(); 1111 private LongSparseArray<Pair<Image, Integer>> mCapturePendingMap = 1112 new LongSparseArray<>(); 1113 1114 private ImageCallback mImageCallback = null; 1115 private boolean mCaptureFailed = false; 1116 private CaptureResultHandler mCaptureResultHandler = null; 1117 BurstRequestHandler(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback callbacks, @NonNull HashMap<CaptureRequest, Integer> requestMap, @Nullable CameraOutputImageCallback imageCallback)1118 public BurstRequestHandler(@NonNull CaptureRequest request, @NonNull Executor executor, 1119 @NonNull ExtensionCaptureCallback callbacks, 1120 @NonNull HashMap<CaptureRequest, Integer> requestMap, 1121 @Nullable CameraOutputImageCallback imageCallback) { 1122 mClientRequest = request; 1123 mExecutor = executor; 1124 mCallbacks = callbacks; 1125 mCaptureRequestMap = requestMap; 1126 mBurstImageCallback = imageCallback; 1127 } 1128 notifyCaptureFailed()1129 private void notifyCaptureFailed() { 1130 if (!mCaptureFailed) { 1131 mCaptureFailed = true; 1132 1133 final long ident = Binder.clearCallingIdentity(); 1134 try { 1135 mExecutor.execute( 1136 () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, 1137 mClientRequest)); 1138 } finally { 1139 Binder.restoreCallingIdentity(ident); 1140 } 1141 1142 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) { 1143 if (captureStage.first != null) { 1144 captureStage.first.close(); 1145 } 1146 } 1147 mCaptureStageMap.clear(); 1148 } 1149 } 1150 1151 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1152 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1153 @NonNull CaptureRequest request, 1154 long timestamp, 1155 long frameNumber) { 1156 // Trigger the client callback only once in case of burst request 1157 boolean initialCallback = false; 1158 synchronized (mInterfaceLock) { 1159 if ((mImageProcessor != null) && (mImageCallback == null)) { 1160 mImageCallback = new ImageCallback(); 1161 initialCallback = true; 1162 } else if (mImageProcessor == null) { 1163 // No burst expected in this case 1164 initialCallback = true; 1165 } 1166 } 1167 1168 if (initialCallback) { 1169 final long ident = Binder.clearCallingIdentity(); 1170 try { 1171 mExecutor.execute( 1172 () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, 1173 mClientRequest, timestamp)); 1174 } finally { 1175 Binder.restoreCallingIdentity(ident); 1176 } 1177 } 1178 1179 if ((mBurstImageCallback != null) && (mImageCallback != null)) { 1180 mBurstImageCallback.registerListener(timestamp, mImageCallback); 1181 } 1182 } 1183 1184 @Override onCaptureBufferLost(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull Surface target, long frameNumber)1185 public void onCaptureBufferLost(@NonNull CameraCaptureSession session, 1186 @NonNull CaptureRequest request, 1187 @NonNull Surface target, long frameNumber) { 1188 notifyCaptureFailed(); 1189 } 1190 1191 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1192 public void onCaptureFailed(@NonNull CameraCaptureSession session, 1193 @NonNull CaptureRequest request, 1194 @NonNull CaptureFailure failure) { 1195 notifyCaptureFailed(); 1196 } 1197 1198 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1199 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 1200 int sequenceId) { 1201 final long ident = Binder.clearCallingIdentity(); 1202 try { 1203 mExecutor.execute( 1204 () -> mCallbacks.onCaptureSequenceAborted(CameraExtensionSessionImpl.this, 1205 sequenceId)); 1206 } finally { 1207 Binder.restoreCallingIdentity(ident); 1208 } 1209 } 1210 1211 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1212 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1213 int sequenceId, 1214 long frameNumber) { 1215 final long ident = Binder.clearCallingIdentity(); 1216 try { 1217 mExecutor.execute( 1218 () -> mCallbacks 1219 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, 1220 sequenceId)); 1221 } finally { 1222 Binder.restoreCallingIdentity(ident); 1223 } 1224 } 1225 1226 @Override onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1227 public void onCaptureCompleted(@NonNull CameraCaptureSession session, 1228 @NonNull CaptureRequest request, 1229 @NonNull TotalCaptureResult result) { 1230 if (!mCaptureRequestMap.containsKey(request)) { 1231 Log.e(TAG, 1232 "Unexpected still capture request received!"); 1233 return; 1234 } 1235 Integer stageId = mCaptureRequestMap.get(request); 1236 1237 Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1238 if (timestamp != null) { 1239 if (mCaptureResultsSupported && (mCaptureResultHandler == null)) { 1240 mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, 1241 mCallbacks, result.getSequenceId()); 1242 } 1243 if (mImageProcessor != null) { 1244 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { 1245 Image img = mCapturePendingMap.get(timestamp).first; 1246 mCapturePendingMap.remove(timestamp); 1247 mCaptureStageMap.put(stageId, new Pair<>(img, result)); 1248 checkAndFireBurstProcessing(); 1249 } else { 1250 mCapturePendingMap.put(timestamp, new Pair<>(null, stageId)); 1251 mCaptureStageMap.put(stageId, new Pair<>(null, result)); 1252 } 1253 } else { 1254 mCaptureRequestMap.clear(); 1255 final long ident = Binder.clearCallingIdentity(); 1256 try { 1257 mExecutor.execute( 1258 () -> mCallbacks 1259 .onCaptureProcessStarted(CameraExtensionSessionImpl.this, 1260 mClientRequest)); 1261 1262 if (mCaptureResultHandler != null) { 1263 mCaptureResultHandler.onCaptureCompleted(timestamp, 1264 initializeFilteredResults(result)); 1265 } 1266 } finally { 1267 Binder.restoreCallingIdentity(ident); 1268 } 1269 } 1270 } else { 1271 Log.e(TAG, 1272 "Capture result without valid sensor timestamp!"); 1273 } 1274 } 1275 checkAndFireBurstProcessing()1276 private void checkAndFireBurstProcessing() { 1277 if (mCaptureRequestMap.size() == mCaptureStageMap.size()) { 1278 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap 1279 .values()) { 1280 if ((captureStage.first == null) || (captureStage.second == null)) { 1281 return; 1282 } 1283 } 1284 1285 mCaptureRequestMap.clear(); 1286 mCapturePendingMap.clear(); 1287 boolean processStatus = true; 1288 Byte jpegQuality = mClientRequest.get(CaptureRequest.JPEG_QUALITY); 1289 Integer jpegOrientation = mClientRequest.get(CaptureRequest.JPEG_ORIENTATION); 1290 List<CaptureBundle> captureList = initializeParcelable(mCaptureStageMap, 1291 jpegOrientation, jpegQuality); 1292 try { 1293 boolean isPostviewRequested = 1294 mClientRequest.containsTarget(mClientPostviewSurface); 1295 mImageProcessor.process(captureList, mCaptureResultHandler, 1296 isPostviewRequested); 1297 } catch (RemoteException e) { 1298 Log.e(TAG, "Failed to process multi-frame request! Extension service " 1299 + "does not respond!"); 1300 processStatus = false; 1301 } 1302 1303 for (CaptureBundle bundle : captureList) { 1304 bundle.captureImage.buffer.close(); 1305 } 1306 captureList.clear(); 1307 for (Pair<Image, TotalCaptureResult> captureStage : mCaptureStageMap.values()) { 1308 captureStage.first.close(); 1309 } 1310 mCaptureStageMap.clear(); 1311 1312 final long ident = Binder.clearCallingIdentity(); 1313 try { 1314 if (processStatus) { 1315 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( 1316 CameraExtensionSessionImpl.this, mClientRequest)); 1317 } else { 1318 mExecutor.execute(() -> mCallbacks.onCaptureFailed( 1319 CameraExtensionSessionImpl.this, mClientRequest)); 1320 } 1321 } finally { 1322 Binder.restoreCallingIdentity(ident); 1323 } 1324 } 1325 } 1326 1327 private class ImageCallback implements OnImageAvailableListener { 1328 @Override onImageDropped(long timestamp)1329 public void onImageDropped(long timestamp) { 1330 notifyCaptureFailed(); 1331 } 1332 1333 @Override onImageAvailable(ImageReader reader, Image img)1334 public void onImageAvailable(ImageReader reader, Image img) { 1335 if (mCaptureFailed) { 1336 img.close(); 1337 } 1338 1339 long timestamp = img.getTimestamp(); 1340 reader.detachImage(img); 1341 if (mCapturePendingMap.indexOfKey(timestamp) >= 0) { 1342 Integer stageId = mCapturePendingMap.get(timestamp).second; 1343 mCapturePendingMap.remove(timestamp); 1344 Pair<Image, TotalCaptureResult> captureStage = 1345 mCaptureStageMap.get(stageId); 1346 if (captureStage != null) { 1347 mCaptureStageMap.put(stageId, 1348 new Pair<>(img, 1349 captureStage.second)); 1350 checkAndFireBurstProcessing(); 1351 } else { 1352 Log.e(TAG, 1353 "Capture stage: " + 1354 mCapturePendingMap.get(timestamp).second + 1355 " is absent!"); 1356 } 1357 } else { 1358 mCapturePendingMap.put(timestamp, 1359 new Pair<>(img, 1360 -1)); 1361 } 1362 } 1363 } 1364 } 1365 1366 private class ImageLoopbackCallback implements OnImageAvailableListener { 1367 @Override onImageDropped(long timestamp)1368 public void onImageDropped(long timestamp) { } 1369 1370 @Override onImageAvailable(ImageReader reader, Image img)1371 public void onImageAvailable(ImageReader reader, Image img) { 1372 img.close(); 1373 } 1374 } 1375 1376 private class InitialRequestHandler extends CameraCaptureSession.CaptureCallback { 1377 private final CameraOutputImageCallback mImageCallback; 1378 InitialRequestHandler(CameraOutputImageCallback imageCallback)1379 public InitialRequestHandler(CameraOutputImageCallback imageCallback) { 1380 mImageCallback = imageCallback; 1381 } 1382 1383 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1384 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1385 @NonNull CaptureRequest request, long timestamp, long frameNumber) { 1386 mImageCallback.registerListener(timestamp, new ImageLoopbackCallback()); 1387 } 1388 1389 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1390 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 1391 int sequenceId) { 1392 Log.e(TAG, "Initial capture request aborted!"); 1393 notifyConfigurationFailure(); 1394 } 1395 1396 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1397 public void onCaptureFailed(@NonNull CameraCaptureSession session, 1398 @NonNull CaptureRequest request, 1399 @NonNull CaptureFailure failure) { 1400 Log.e(TAG, "Initial capture request failed!"); 1401 notifyConfigurationFailure(); 1402 } 1403 1404 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1405 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1406 int sequenceId, 1407 long frameNumber) { 1408 boolean status = true; 1409 synchronized (mInterfaceLock) { 1410 /** 1411 * Initialize and set the initial repeating request which will execute in the 1412 * absence of client repeating requests. 1413 */ 1414 try { 1415 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 1416 new PreviewRequestHandler(null, null, null, 1417 mImageCallback)); 1418 } catch (CameraAccessException | RemoteException e) { 1419 Log.e(TAG, "Failed to start the internal repeating request!"); 1420 status = false; 1421 } 1422 1423 } 1424 1425 if (!status) { 1426 notifyConfigurationFailure(); 1427 } 1428 } 1429 } 1430 1431 private interface OnImageAvailableListener { onImageDropped(long timestamp)1432 void onImageDropped(long timestamp); onImageAvailable(ImageReader reader, Image img)1433 void onImageAvailable (ImageReader reader, Image img); 1434 } 1435 1436 private class CameraOutputImageCallback implements ImageReader.OnImageAvailableListener, 1437 Closeable { 1438 private final ImageReader mImageReader; 1439 // Map timestamp to specific images and listeners 1440 private HashMap<Long, Pair<Image, OnImageAvailableListener>> mImageListenerMap = 1441 new HashMap<>(); 1442 private boolean mOutOfBuffers = false; 1443 private final boolean mPruneOlderBuffers; 1444 CameraOutputImageCallback(ImageReader imageReader, boolean pruneOlderBuffers)1445 CameraOutputImageCallback(ImageReader imageReader, boolean pruneOlderBuffers) { 1446 mImageReader = imageReader; 1447 mPruneOlderBuffers = pruneOlderBuffers; 1448 } 1449 1450 @Override onImageAvailable(ImageReader reader)1451 public void onImageAvailable(ImageReader reader) { 1452 Image img; 1453 synchronized (mInterfaceLock) { 1454 try { 1455 img = reader.acquireNextImage(); 1456 } catch (IllegalStateException e) { 1457 Log.e(TAG, "Failed to acquire image, too many images pending!"); 1458 mOutOfBuffers = true; 1459 return; 1460 } 1461 if (img == null) { 1462 Log.e(TAG, "Invalid image!"); 1463 return; 1464 } 1465 1466 Long timestamp = img.getTimestamp(); 1467 if (mImageListenerMap.containsKey(timestamp)) { 1468 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove( 1469 timestamp); 1470 if (entry.second != null) { 1471 entry.second.onImageAvailable(reader, img); 1472 } else { 1473 Log.w(TAG, "Invalid image listener, dropping frame!"); 1474 img.close(); 1475 } 1476 } else { 1477 mImageListenerMap.put(timestamp, new Pair<>(img, null)); 1478 } 1479 1480 notifyDroppedImages(timestamp); 1481 } 1482 } 1483 notifyDroppedImages(long timestamp)1484 private void notifyDroppedImages(long timestamp) { 1485 synchronized (mInterfaceLock) { 1486 Set<Long> timestamps = mImageListenerMap.keySet(); 1487 ArrayList<Long> removedTs = new ArrayList<>(); 1488 for (long ts : timestamps) { 1489 if (ts < timestamp) { 1490 if (!mPruneOlderBuffers) { 1491 Log.w(TAG, "Unexpected older image with ts: " + ts); 1492 continue; 1493 } 1494 Log.e(TAG, "Dropped image with ts: " + ts); 1495 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(ts); 1496 if (entry.second != null) { 1497 entry.second.onImageDropped(ts); 1498 } 1499 if (entry.first != null) { 1500 entry.first.close(); 1501 } 1502 removedTs.add(ts); 1503 } 1504 } 1505 for (long ts : removedTs) { 1506 mImageListenerMap.remove(ts); 1507 } 1508 } 1509 } 1510 registerListener(Long timestamp, OnImageAvailableListener listener)1511 public void registerListener(Long timestamp, OnImageAvailableListener listener) { 1512 synchronized (mInterfaceLock) { 1513 if (mImageListenerMap.containsKey(timestamp)) { 1514 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.remove( 1515 timestamp); 1516 if (entry.first != null) { 1517 listener.onImageAvailable(mImageReader, entry.first); 1518 if (mOutOfBuffers) { 1519 mOutOfBuffers = false; 1520 Log.w(TAG,"Out of buffers, retry!"); 1521 onImageAvailable(mImageReader); 1522 } 1523 } else { 1524 Log.w(TAG, "No valid image for listener with ts: " + 1525 timestamp.longValue()); 1526 } 1527 } else { 1528 mImageListenerMap.put(timestamp, new Pair<>(null, listener)); 1529 } 1530 } 1531 } 1532 1533 @Override close()1534 public void close() { 1535 synchronized (mInterfaceLock) { 1536 for (Pair<Image, OnImageAvailableListener> entry : mImageListenerMap.values()) { 1537 if (entry.first != null) { 1538 entry.first.close(); 1539 } 1540 } 1541 for (long timestamp : mImageListenerMap.keySet()) { 1542 Pair<Image, OnImageAvailableListener> entry = mImageListenerMap.get(timestamp); 1543 if (entry.second != null) { 1544 entry.second.onImageDropped(timestamp); 1545 } 1546 } 1547 mImageListenerMap.clear(); 1548 } 1549 } 1550 } 1551 1552 private class CloseRequestHandler extends CameraCaptureSession.CaptureCallback { 1553 private final CameraOutputImageCallback mImageCallback; 1554 CloseRequestHandler(CameraOutputImageCallback imageCallback)1555 public CloseRequestHandler(CameraOutputImageCallback imageCallback) { 1556 mImageCallback = imageCallback; 1557 } 1558 1559 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1560 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1561 @NonNull CaptureRequest request, long timestamp, long frameNumber) { 1562 mImageCallback.registerListener(timestamp, new ImageLoopbackCallback()); 1563 } 1564 } 1565 1566 private class CaptureResultHandler extends IProcessResultImpl.Stub { 1567 private final Executor mExecutor; 1568 private final ExtensionCaptureCallback mCallbacks; 1569 private final CaptureRequest mClientRequest; 1570 private final int mRequestId; 1571 CaptureResultHandler(@onNull CaptureRequest clientRequest, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, int requestId)1572 public CaptureResultHandler(@NonNull CaptureRequest clientRequest, 1573 @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener, 1574 int requestId) { 1575 mClientRequest = clientRequest; 1576 mExecutor = executor; 1577 mCallbacks = listener; 1578 mRequestId = requestId; 1579 } 1580 1581 @Override onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result)1582 public void onCaptureCompleted(long shutterTimestamp, CameraMetadataNative result) { 1583 if (result == null) { 1584 Log.e(TAG,"Invalid capture result!"); 1585 return; 1586 } 1587 1588 result.set(CaptureResult.SENSOR_TIMESTAMP, shutterTimestamp); 1589 TotalCaptureResult totalResult = new TotalCaptureResult(mCameraDevice.getId(), result, 1590 mClientRequest, mRequestId, shutterTimestamp, new ArrayList<CaptureResult>(), 1591 mSessionId, new PhysicalCaptureResultInfo[0]); 1592 final long ident = Binder.clearCallingIdentity(); 1593 try { 1594 mExecutor.execute( 1595 () -> mCallbacks.onCaptureResultAvailable(CameraExtensionSessionImpl.this, 1596 mClientRequest, totalResult)); 1597 } finally { 1598 Binder.restoreCallingIdentity(ident); 1599 } 1600 } 1601 1602 @Override onCaptureProcessProgressed(int progress)1603 public void onCaptureProcessProgressed(int progress) { 1604 final long ident = Binder.clearCallingIdentity(); 1605 try { 1606 mExecutor.execute( 1607 () -> mCallbacks.onCaptureProcessProgressed(CameraExtensionSessionImpl.this, 1608 mClientRequest, progress)); 1609 } finally { 1610 Binder.restoreCallingIdentity(ident); 1611 } 1612 } 1613 } 1614 1615 // This handler can operate in three modes: 1616 // 1) Using valid client callbacks, which means camera buffers will be propagated the 1617 // registered output surfaces and clients will be notified accordingly. 1618 // 2) Without any client callbacks where an internal repeating request is kept active 1619 // to satisfy the extensions continuous preview/(repeating request) requirement. 1620 // 3) Single capture mode, where internal repeating requests are ignored and the preview 1621 // logic is only triggered for the image processor case. 1622 private class PreviewRequestHandler extends CameraCaptureSession.CaptureCallback { 1623 private final Executor mExecutor; 1624 private final ExtensionCaptureCallback mCallbacks; 1625 private final CaptureRequest mClientRequest; 1626 private final boolean mClientNotificationsEnabled; 1627 private final CameraOutputImageCallback mRepeatingImageCallback; 1628 private final boolean mSingleCapture; 1629 private OnImageAvailableListener mImageCallback = null; 1630 private LongSparseArray<Pair<Image, TotalCaptureResult>> mPendingResultMap = 1631 new LongSparseArray<>(); 1632 private CaptureResultHandler mCaptureResultHandler = null; 1633 1634 private boolean mRequestUpdatedNeeded = false; 1635 PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback)1636 public PreviewRequestHandler(@Nullable CaptureRequest clientRequest, 1637 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, 1638 @NonNull CameraOutputImageCallback imageCallback) { 1639 this(clientRequest, executor, listener, imageCallback, false /*singleCapture*/); 1640 } 1641 PreviewRequestHandler(@ullable CaptureRequest clientRequest, @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture)1642 public PreviewRequestHandler(@Nullable CaptureRequest clientRequest, 1643 @Nullable Executor executor, @Nullable ExtensionCaptureCallback listener, 1644 @NonNull CameraOutputImageCallback imageCallback, boolean singleCapture) { 1645 mClientRequest = clientRequest; 1646 mExecutor = executor; 1647 mCallbacks = listener; 1648 mClientNotificationsEnabled = 1649 (mClientRequest != null) && (mExecutor != null) && (mCallbacks != null); 1650 mRepeatingImageCallback = imageCallback; 1651 mSingleCapture = singleCapture; 1652 } 1653 1654 @Override onCaptureStarted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber)1655 public void onCaptureStarted(@NonNull CameraCaptureSession session, 1656 @NonNull CaptureRequest request, 1657 long timestamp, 1658 long frameNumber) { 1659 synchronized (mInterfaceLock) { 1660 // Setup the image callback handler for this repeating request just once 1661 // after streaming resumes. 1662 if (mImageCallback == null) { 1663 if (mPreviewProcessorType == 1664 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) { 1665 if (mClientNotificationsEnabled) { 1666 mPreviewImageProcessor.onOutputSurface(mClientRepeatingRequestSurface, 1667 nativeGetSurfaceFormat(mClientRepeatingRequestSurface)); 1668 } else { 1669 mPreviewImageProcessor.onOutputSurface(null, -1); 1670 } 1671 mImageCallback = new ImageProcessCallback(); 1672 } else { 1673 mImageCallback = mClientNotificationsEnabled ? 1674 new ImageForwardCallback(mRepeatingRequestImageWriter) : 1675 new ImageLoopbackCallback(); 1676 } 1677 } 1678 } 1679 1680 if (mClientNotificationsEnabled) { 1681 final long ident = Binder.clearCallingIdentity(); 1682 try { 1683 mExecutor.execute( 1684 () -> mCallbacks.onCaptureStarted(CameraExtensionSessionImpl.this, 1685 mClientRequest, timestamp)); 1686 } finally { 1687 Binder.restoreCallingIdentity(ident); 1688 } 1689 } 1690 1691 mRepeatingImageCallback.registerListener(timestamp, mImageCallback); 1692 } 1693 1694 @Override onCaptureSequenceAborted(@onNull CameraCaptureSession session, int sequenceId)1695 public void onCaptureSequenceAborted(@NonNull CameraCaptureSession session, 1696 int sequenceId) { 1697 synchronized (mInterfaceLock) { 1698 if (mInternalRepeatingRequestEnabled && !mSingleCapture) { 1699 resumeInternalRepeatingRequest(true); 1700 } 1701 } 1702 1703 if (mClientNotificationsEnabled) { 1704 final long ident = Binder.clearCallingIdentity(); 1705 try { 1706 mExecutor.execute( 1707 () -> mCallbacks 1708 .onCaptureSequenceAborted(CameraExtensionSessionImpl.this, 1709 sequenceId)); 1710 } finally { 1711 Binder.restoreCallingIdentity(ident); 1712 } 1713 } else { 1714 notifyConfigurationFailure(); 1715 } 1716 } 1717 1718 @Override onCaptureSequenceCompleted(@onNull CameraCaptureSession session, int sequenceId, long frameNumber)1719 public void onCaptureSequenceCompleted(@NonNull CameraCaptureSession session, 1720 int sequenceId, 1721 long frameNumber) { 1722 1723 synchronized (mInterfaceLock) { 1724 if (mRequestUpdatedNeeded && !mSingleCapture) { 1725 mRequestUpdatedNeeded = false; 1726 resumeInternalRepeatingRequest(false); 1727 } else if (mInternalRepeatingRequestEnabled && !mSingleCapture) { 1728 resumeInternalRepeatingRequest(true); 1729 } 1730 } 1731 1732 if (mClientNotificationsEnabled) { 1733 final long ident = Binder.clearCallingIdentity(); 1734 try { 1735 mExecutor.execute( 1736 () -> mCallbacks 1737 .onCaptureSequenceCompleted(CameraExtensionSessionImpl.this, 1738 sequenceId)); 1739 } finally { 1740 Binder.restoreCallingIdentity(ident); 1741 } 1742 } 1743 } 1744 1745 @Override onCaptureFailed(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull CaptureFailure failure)1746 public void onCaptureFailed(@NonNull CameraCaptureSession session, 1747 @NonNull CaptureRequest request, 1748 @NonNull CaptureFailure failure) { 1749 1750 if (mClientNotificationsEnabled) { 1751 final long ident = Binder.clearCallingIdentity(); 1752 try { 1753 mExecutor.execute( 1754 () -> mCallbacks.onCaptureFailed(CameraExtensionSessionImpl.this, 1755 mClientRequest)); 1756 } finally { 1757 Binder.restoreCallingIdentity(ident); 1758 } 1759 } 1760 } 1761 1762 @Override onCaptureCompleted(@onNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)1763 public void onCaptureCompleted(@NonNull CameraCaptureSession session, 1764 @NonNull CaptureRequest request, 1765 @NonNull TotalCaptureResult result) { 1766 boolean notifyClient = mClientNotificationsEnabled; 1767 boolean processStatus = true; 1768 1769 synchronized (mInterfaceLock) { 1770 final Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1771 if (timestamp != null) { 1772 if (mCaptureResultsSupported && mClientNotificationsEnabled && 1773 (mCaptureResultHandler == null)) { 1774 mCaptureResultHandler = new CaptureResultHandler(mClientRequest, mExecutor, 1775 mCallbacks, result.getSequenceId()); 1776 } 1777 if ((!mSingleCapture) && (mPreviewProcessorType == 1778 IPreviewExtenderImpl.PROCESSOR_TYPE_REQUEST_UPDATE_ONLY) 1779 && mInitialized) { 1780 CaptureStageImpl captureStage = null; 1781 try { 1782 captureStage = mPreviewRequestUpdateProcessor.process( 1783 result.getNativeMetadata(), result.getSequenceId()); 1784 } catch (RemoteException e) { 1785 Log.e(TAG, "Extension service does not respond during " + 1786 "processing!"); 1787 } 1788 if (captureStage != null) { 1789 try { 1790 setRepeatingRequest(captureStage, this, request); 1791 mRequestUpdatedNeeded = true; 1792 } catch (IllegalStateException e) { 1793 // This is possible in case the camera device closes and the 1794 // and the callback here is executed before the onClosed 1795 // notification. 1796 } catch (CameraAccessException e) { 1797 Log.e(TAG, "Failed to update repeating request settings!"); 1798 } 1799 } else { 1800 mRequestUpdatedNeeded = false; 1801 } 1802 } else if ((mPreviewProcessorType == 1803 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR) && mInitialized) { 1804 int idx = mPendingResultMap.indexOfKey(timestamp); 1805 1806 if ((idx >= 0) && (mPendingResultMap.get(timestamp).first == null)) { 1807 // Image was dropped before we can receive the capture results 1808 if ((mCaptureResultHandler != null)) { 1809 mCaptureResultHandler.onCaptureCompleted(timestamp, 1810 initializeFilteredResults(result)); 1811 } 1812 discardPendingRepeatingResults(idx, mPendingResultMap, false); 1813 } else if (idx >= 0) { 1814 // Image came before the capture results 1815 ParcelImage parcelImage = initializeParcelImage( 1816 mPendingResultMap.get(timestamp).first); 1817 try { 1818 mPreviewImageProcessor.process(parcelImage, result, 1819 mCaptureResultHandler); 1820 } catch (RemoteException e) { 1821 processStatus = false; 1822 Log.e(TAG, "Extension service does not respond during " + 1823 "processing, dropping frame!"); 1824 } catch (RuntimeException e) { 1825 // Runtime exceptions can happen in a few obscure cases where the 1826 // client tries to initialize a new capture session while this 1827 // session is still ongoing. In such scenario, the camera will 1828 // disconnect from the intermediate output surface, which will 1829 // invalidate the images that we acquired previously. This can 1830 // happen before we get notified via "onClosed" so there aren't 1831 // many options to avoid the exception. 1832 processStatus = false; 1833 Log.e(TAG, "Runtime exception encountered during buffer " + 1834 "processing, dropping frame!"); 1835 } finally { 1836 parcelImage.buffer.close(); 1837 mPendingResultMap.get(timestamp).first.close(); 1838 } 1839 discardPendingRepeatingResults(idx, mPendingResultMap, false); 1840 } else { 1841 // Image not yet available 1842 notifyClient = false; 1843 mPendingResultMap.put(timestamp, 1844 new Pair<>(null, 1845 result)); 1846 } 1847 } else { 1848 // No special handling for PROCESSOR_TYPE_NONE 1849 } 1850 if (notifyClient && mInitialized) { 1851 final long ident = Binder.clearCallingIdentity(); 1852 try { 1853 if (processStatus) { 1854 mExecutor.execute(() -> mCallbacks 1855 .onCaptureProcessStarted( 1856 CameraExtensionSessionImpl.this, 1857 mClientRequest)); 1858 if ((mCaptureResultHandler != null) && (mPreviewProcessorType != 1859 IPreviewExtenderImpl.PROCESSOR_TYPE_IMAGE_PROCESSOR)) { 1860 mCaptureResultHandler.onCaptureCompleted(timestamp, 1861 initializeFilteredResults(result)); 1862 } 1863 } else { 1864 mExecutor.execute( 1865 () -> mCallbacks 1866 .onCaptureFailed( 1867 CameraExtensionSessionImpl.this, 1868 mClientRequest)); 1869 } 1870 } finally { 1871 Binder.restoreCallingIdentity(ident); 1872 } 1873 } 1874 } else { 1875 Log.e(TAG, 1876 "Result without valid sensor timestamp!"); 1877 } 1878 } 1879 1880 if (!notifyClient) { 1881 notifyConfigurationSuccess(); 1882 } 1883 } 1884 resumeInternalRepeatingRequest(boolean internal)1885 private void resumeInternalRepeatingRequest(boolean internal) { 1886 try { 1887 if (internal) { 1888 setRepeatingRequest(mPreviewExtender.getCaptureStage(), 1889 new PreviewRequestHandler(null, null, null, 1890 mRepeatingImageCallback)); 1891 } else { 1892 setRepeatingRequest(mPreviewExtender.getCaptureStage(), this, mClientRequest); 1893 } 1894 } catch (RemoteException e) { 1895 Log.e(TAG, "Failed to resume internal repeating request, extension service" 1896 + " fails to respond!"); 1897 } catch (IllegalStateException e) { 1898 // This is possible in case we try to resume before the state "onClosed" 1899 // notification is able to reach us. 1900 Log.w(TAG, "Failed to resume internal repeating request!"); 1901 } catch (CameraAccessException e) { 1902 Log.e(TAG, "Failed to resume internal repeating request!"); 1903 } 1904 } 1905 1906 // Find the timestamp of the oldest pending buffer calculatePruneThreshold( LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap)1907 private Long calculatePruneThreshold( 1908 LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap) { 1909 long oldestTimestamp = Long.MAX_VALUE; 1910 for (int idx = 0; idx < previewMap.size(); idx++) { 1911 Pair<Image, TotalCaptureResult> entry = previewMap.valueAt(idx); 1912 long timestamp = previewMap.keyAt(idx); 1913 if ((entry.first != null) && (timestamp < oldestTimestamp)) { 1914 oldestTimestamp = timestamp; 1915 } 1916 } 1917 return (oldestTimestamp == Long.MAX_VALUE) ? 0 : oldestTimestamp; 1918 } 1919 discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, TotalCaptureResult>> previewMap, boolean notifyCurrentIndex)1920 private void discardPendingRepeatingResults(int idx, LongSparseArray<Pair<Image, 1921 TotalCaptureResult>> previewMap, boolean notifyCurrentIndex) { 1922 if (idx < 0) { 1923 return; 1924 } 1925 for (int i = idx; i >= 0; i--) { 1926 if (previewMap.valueAt(i).first != null) { 1927 previewMap.valueAt(i).first.close(); 1928 } else if (mClientNotificationsEnabled && (previewMap.valueAt(i).second != null) && 1929 ((i != idx) || notifyCurrentIndex)) { 1930 TotalCaptureResult result = previewMap.valueAt(i).second; 1931 Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 1932 if (mCaptureResultHandler != null) { 1933 mCaptureResultHandler.onCaptureCompleted(timestamp, 1934 initializeFilteredResults(result)); 1935 } 1936 1937 Log.w(TAG, "Preview frame drop with timestamp: " + previewMap.keyAt(i)); 1938 final long ident = Binder.clearCallingIdentity(); 1939 try { 1940 mExecutor.execute( 1941 () -> mCallbacks 1942 .onCaptureFailed(CameraExtensionSessionImpl.this, 1943 mClientRequest)); 1944 } finally { 1945 Binder.restoreCallingIdentity(ident); 1946 } 1947 1948 } 1949 previewMap.removeAt(i); 1950 } 1951 } 1952 1953 private class ImageForwardCallback implements OnImageAvailableListener { 1954 private final ImageWriter mOutputWriter; 1955 ImageForwardCallback(@onNull ImageWriter imageWriter)1956 public ImageForwardCallback(@NonNull ImageWriter imageWriter) { 1957 mOutputWriter = imageWriter; 1958 } 1959 1960 @Override onImageDropped(long timestamp)1961 public void onImageDropped(long timestamp) { 1962 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), 1963 mPendingResultMap, true); 1964 } 1965 1966 @Override onImageAvailable(ImageReader reader, Image img)1967 public void onImageAvailable(ImageReader reader, Image img) { 1968 if (img == null) { 1969 Log.e(TAG, "Invalid image!"); 1970 return; 1971 } 1972 1973 try { 1974 mOutputWriter.queueInputImage(img); 1975 } catch (IllegalStateException e) { 1976 // This is possible in case the client disconnects from the output surface 1977 // abruptly. 1978 Log.w(TAG, "Output surface likely abandoned, dropping buffer!"); 1979 img.close(); 1980 } catch (RuntimeException e) { 1981 // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage 1982 // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the 1983 // bug went unchecked for a few years and now its behavior cannot be changed 1984 // without breaking backwards compatibility. 1985 1986 if (!e.getClass().equals(RuntimeException.class)) { 1987 // re-throw any exceptions that aren't base RuntimeException since they are 1988 // coming from elsewhere, and we shouldn't silently drop those. 1989 throw e; 1990 } 1991 1992 Log.w(TAG, "Output surface likely abandoned, dropping buffer!"); 1993 img.close(); 1994 } 1995 } 1996 } 1997 1998 private class ImageProcessCallback implements OnImageAvailableListener { 1999 2000 @Override onImageDropped(long timestamp)2001 public void onImageDropped(long timestamp) { 2002 discardPendingRepeatingResults(mPendingResultMap.indexOfKey(timestamp), 2003 mPendingResultMap, true); 2004 // Add an empty frame&results entry to flag that we dropped a frame 2005 // and valid capture results can immediately return to client. 2006 mPendingResultMap.put(timestamp, new Pair<>(null, null)); 2007 } 2008 2009 @Override onImageAvailable(ImageReader reader, Image img)2010 public void onImageAvailable(ImageReader reader, Image img) { 2011 if (mPendingResultMap.size() + 1 >= PREVIEW_QUEUE_SIZE) { 2012 // We reached the maximum acquired images limit. This is possible in case we 2013 // have capture failures that result in absent or missing capture results. In 2014 // such scenario we can prune the oldest pending buffer. 2015 discardPendingRepeatingResults( 2016 mPendingResultMap 2017 .indexOfKey(calculatePruneThreshold(mPendingResultMap)), 2018 mPendingResultMap, true); 2019 } 2020 2021 if (img == null) { 2022 Log.e(TAG, 2023 "Invalid preview buffer!"); 2024 return; 2025 } 2026 try { 2027 reader.detachImage(img); 2028 } catch (IllegalStateException e) { 2029 Log.e(TAG, "Failed to detach image!"); 2030 img.close(); 2031 return; 2032 } catch (RuntimeException e) { 2033 // NOTE: This is intended to catch RuntimeException from ImageReader.detachImage 2034 // ImageReader.detachImage is not supposed to throw RuntimeExceptions but the 2035 // bug went unchecked for a few years and now its behavior cannot be changed 2036 // without breaking backwards compatibility. 2037 2038 if (!e.getClass().equals(RuntimeException.class)) { 2039 // re-throw any exceptions that aren't base RuntimeException since they are 2040 // coming from elsewhere, and we shouldn't silently drop those. 2041 throw e; 2042 } 2043 2044 Log.e(TAG, "Failed to detach image!"); 2045 img.close(); 2046 return; 2047 } 2048 2049 long timestamp = img.getTimestamp(); 2050 int idx = mPendingResultMap.indexOfKey(timestamp); 2051 if (idx >= 0) { 2052 boolean processStatus = true; 2053 ParcelImage parcelImage = initializeParcelImage(img); 2054 try { 2055 mPreviewImageProcessor.process(parcelImage, 2056 mPendingResultMap.get(timestamp).second, mCaptureResultHandler); 2057 } catch (RemoteException e) { 2058 processStatus = false; 2059 Log.e(TAG, "Extension service does not respond during " + 2060 "processing, dropping frame!"); 2061 } finally { 2062 parcelImage.buffer.close(); 2063 img.close(); 2064 } 2065 discardPendingRepeatingResults(idx, mPendingResultMap, false); 2066 if (mClientNotificationsEnabled) { 2067 final long ident = Binder.clearCallingIdentity(); 2068 try { 2069 if (processStatus) { 2070 mExecutor.execute(() -> mCallbacks.onCaptureProcessStarted( 2071 CameraExtensionSessionImpl.this, 2072 mClientRequest)); 2073 } else { 2074 mExecutor.execute(() -> mCallbacks.onCaptureFailed( 2075 CameraExtensionSessionImpl.this, 2076 mClientRequest)); 2077 } 2078 } finally { 2079 Binder.restoreCallingIdentity(ident); 2080 } 2081 } 2082 } else { 2083 mPendingResultMap.put(timestamp, new Pair<>(img, null)); 2084 } 2085 } 2086 } 2087 } 2088 initializeFilteredResults(TotalCaptureResult result)2089 private CameraMetadataNative initializeFilteredResults(TotalCaptureResult result) { 2090 CameraMetadataNative captureResults = new CameraMetadataNative(); 2091 for (CaptureResult.Key key : mSupportedResultKeys) { 2092 Object value = result.get(key); 2093 if (value != null) { 2094 captureResults.set(key, value); 2095 } 2096 } 2097 return captureResults; 2098 } 2099 findSmallestAspectMatchedSize(@onNull List<Size> sizes, @NonNull Size arSize)2100 private static Size findSmallestAspectMatchedSize(@NonNull List<Size> sizes, 2101 @NonNull Size arSize) { 2102 final float TOLL = .01f; 2103 2104 if (arSize.getHeight() == 0) { 2105 throw new IllegalArgumentException("Invalid input aspect ratio"); 2106 } 2107 2108 float targetAR = ((float) arSize.getWidth()) / arSize.getHeight(); 2109 Size ret = null; 2110 Size fallbackSize = null; 2111 for (Size sz : sizes) { 2112 if (fallbackSize == null) { 2113 fallbackSize = sz; 2114 } 2115 if ((sz.getHeight() > 0) && 2116 ((ret == null) || 2117 (ret.getWidth() * ret.getHeight()) < 2118 (sz.getWidth() * sz.getHeight()))) { 2119 float currentAR = ((float) sz.getWidth()) / sz.getHeight(); 2120 if (Math.abs(currentAR - targetAR) <= TOLL) { 2121 ret = sz; 2122 } 2123 } 2124 } 2125 if (ret == null) { 2126 Log.e(TAG, "AR matched size not found returning first size in list"); 2127 ret = fallbackSize; 2128 } 2129 2130 return ret; 2131 } 2132 initializeParcelImage(Image img)2133 private static ParcelImage initializeParcelImage(Image img) { 2134 ParcelImage parcelImage = new ParcelImage(); 2135 parcelImage.buffer = img.getHardwareBuffer(); 2136 try { 2137 SyncFence fd = img.getFence(); 2138 if (fd.isValid()) { 2139 parcelImage.fence = fd.getFdDup(); 2140 } 2141 } catch (IOException e) { 2142 Log.e(TAG, "Failed to parcel buffer fence!"); 2143 } 2144 parcelImage.width = img.getWidth(); 2145 parcelImage.height = img.getHeight(); 2146 parcelImage.format = img.getFormat(); 2147 parcelImage.timestamp = img.getTimestamp(); 2148 parcelImage.transform = img.getTransform(); 2149 parcelImage.scalingMode = img.getScalingMode(); 2150 parcelImage.planeCount = img.getPlaneCount(); 2151 parcelImage.crop = img.getCropRect(); 2152 2153 return parcelImage; 2154 } 2155 initializeParcelable( HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, Byte jpegQuality)2156 private static List<CaptureBundle> initializeParcelable( 2157 HashMap<Integer, Pair<Image, TotalCaptureResult>> captureMap, Integer jpegOrientation, 2158 Byte jpegQuality) { 2159 ArrayList<CaptureBundle> ret = new ArrayList<>(); 2160 for (Integer stageId : captureMap.keySet()) { 2161 Pair<Image, TotalCaptureResult> entry = captureMap.get(stageId); 2162 CaptureBundle bundle = new CaptureBundle(); 2163 bundle.stage = stageId; 2164 bundle.captureImage = initializeParcelImage(entry.first); 2165 bundle.sequenceId = entry.second.getSequenceId(); 2166 bundle.captureResult = entry.second.getNativeMetadata(); 2167 if (jpegOrientation != null) { 2168 bundle.captureResult.set(CaptureResult.JPEG_ORIENTATION, jpegOrientation); 2169 } 2170 if (jpegQuality != null) { 2171 bundle.captureResult.set(CaptureResult.JPEG_QUALITY, jpegQuality); 2172 } 2173 ret.add(bundle); 2174 } 2175 2176 return ret; 2177 } 2178 } 2179