1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import android.annotation.NonNull; 20 import android.annotation.Nullable; 21 import android.annotation.RequiresPermission; 22 import android.content.Context; 23 import android.graphics.ImageFormat; 24 import android.graphics.SurfaceTexture; 25 import android.hardware.SyncFence; 26 import android.hardware.camera2.CameraAccessException; 27 import android.hardware.camera2.CameraCaptureSession; 28 import android.hardware.camera2.CameraCharacteristics; 29 import android.hardware.camera2.CameraDevice; 30 import android.hardware.camera2.CameraExtensionCharacteristics; 31 import android.hardware.camera2.CameraExtensionSession; 32 import android.hardware.camera2.CameraManager; 33 import android.hardware.camera2.CaptureFailure; 34 import android.hardware.camera2.CaptureRequest; 35 import android.hardware.camera2.CaptureResult; 36 import android.hardware.camera2.TotalCaptureResult; 37 import android.hardware.camera2.extension.CameraOutputConfig; 38 import android.hardware.camera2.extension.CameraSessionConfig; 39 import android.hardware.camera2.extension.IAdvancedExtenderImpl; 40 import android.hardware.camera2.extension.ICaptureCallback; 41 import android.hardware.camera2.extension.IImageProcessorImpl; 42 import android.hardware.camera2.extension.IInitializeSessionCallback; 43 import android.hardware.camera2.extension.IRequestCallback; 44 import android.hardware.camera2.extension.IRequestProcessorImpl; 45 import android.hardware.camera2.extension.ISessionProcessorImpl; 46 import android.hardware.camera2.extension.OutputConfigId; 47 import android.hardware.camera2.extension.OutputSurface; 48 import android.hardware.camera2.extension.ParcelCaptureResult; 49 import android.hardware.camera2.extension.ParcelImage; 50 import android.hardware.camera2.extension.ParcelTotalCaptureResult; 51 import android.hardware.camera2.extension.Request; 52 import android.hardware.camera2.params.DynamicRangeProfiles; 53 import android.hardware.camera2.params.ExtensionSessionConfiguration; 54 import android.hardware.camera2.params.OutputConfiguration; 55 import android.hardware.camera2.params.SessionConfiguration; 56 import android.hardware.camera2.utils.SurfaceUtils; 57 import android.media.Image; 58 import android.media.ImageReader; 59 import android.os.Binder; 60 import android.os.Handler; 61 import android.os.HandlerThread; 62 import android.os.RemoteException; 63 import android.util.Log; 64 import android.util.Size; 65 import android.view.Surface; 66 67 import java.io.IOException; 68 import java.util.ArrayList; 69 import java.util.HashMap; 70 import java.util.List; 71 import java.util.Map; 72 import java.util.concurrent.Executor; 73 74 public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSession { 75 private static final String TAG = "CameraAdvancedExtensionSessionImpl"; 76 77 private final Executor mExecutor; 78 private final CameraDevice mCameraDevice; 79 private final long mExtensionClientId; 80 private final Handler mHandler; 81 private final HandlerThread mHandlerThread; 82 private final CameraExtensionSession.StateCallback mCallbacks; 83 private final IAdvancedExtenderImpl mAdvancedExtender; 84 // maps registered camera surfaces to extension output configs 85 private final HashMap<Surface, CameraOutputConfig> mCameraConfigMap = new HashMap<>(); 86 // maps camera extension output ids to camera registered image readers 87 private final HashMap<Integer, ImageReader> mReaderMap = new HashMap<>(); 88 private final RequestProcessor mRequestProcessor = new RequestProcessor(); 89 private final int mSessionId; 90 91 private Surface mClientRepeatingRequestSurface; 92 private Surface mClientCaptureSurface; 93 private CameraCaptureSession mCaptureSession = null; 94 private ISessionProcessorImpl mSessionProcessor = null; 95 private final InitializeSessionHandler mInitializeHandler; 96 97 private boolean mInitialized; 98 99 100 // Lock to synchronize cross-thread access to device public interface 101 final Object mInterfaceLock = new Object(); // access from this class and Session only! 102 103 /** 104 * @hide 105 */ 106 @RequiresPermission(android.Manifest.permission.CAMERA) createCameraAdvancedExtensionSession( @onNull CameraDevice cameraDevice, @NonNull Context ctx, @NonNull ExtensionSessionConfiguration config, int sessionId)107 public static CameraAdvancedExtensionSessionImpl createCameraAdvancedExtensionSession( 108 @NonNull CameraDevice cameraDevice, @NonNull Context ctx, 109 @NonNull ExtensionSessionConfiguration config, int sessionId) 110 throws CameraAccessException, RemoteException { 111 long clientId = CameraExtensionCharacteristics.registerClient(ctx); 112 if (clientId < 0) { 113 throw new UnsupportedOperationException("Unsupported extension!"); 114 } 115 116 String cameraId = cameraDevice.getId(); 117 CameraManager manager = ctx.getSystemService(CameraManager.class); 118 CameraCharacteristics chars = manager.getCameraCharacteristics(cameraId); 119 CameraExtensionCharacteristics extensionChars = new CameraExtensionCharacteristics(ctx, 120 cameraId, chars); 121 122 if (!CameraExtensionCharacteristics.isExtensionSupported(cameraDevice.getId(), 123 config.getExtension(), chars)) { 124 throw new UnsupportedOperationException("Unsupported extension type: " + 125 config.getExtension()); 126 } 127 128 if (config.getOutputConfigurations().isEmpty() || 129 config.getOutputConfigurations().size() > 2) { 130 throw new IllegalArgumentException("Unexpected amount of output surfaces, received: " + 131 config.getOutputConfigurations().size() + " expected <= 2"); 132 } 133 134 for (OutputConfiguration c : config.getOutputConfigurations()) { 135 if (c.getDynamicRangeProfile() != DynamicRangeProfiles.STANDARD) { 136 throw new IllegalArgumentException("Unsupported dynamic range profile: " + 137 c.getDynamicRangeProfile()); 138 } 139 if (c.getStreamUseCase() != 140 CameraCharacteristics.SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT) { 141 throw new IllegalArgumentException("Unsupported stream use case: " + 142 c.getStreamUseCase()); 143 } 144 } 145 146 int suitableSurfaceCount = 0; 147 List<Size> supportedPreviewSizes = extensionChars.getExtensionSupportedSizes( 148 config.getExtension(), SurfaceTexture.class); 149 Surface repeatingRequestSurface = CameraExtensionUtils.getRepeatingRequestSurface( 150 config.getOutputConfigurations(), supportedPreviewSizes); 151 if (repeatingRequestSurface != null) { 152 suitableSurfaceCount++; 153 } 154 155 HashMap<Integer, List<Size>> supportedCaptureSizes = new HashMap<>(); 156 for (int format : CameraExtensionUtils.SUPPORTED_CAPTURE_OUTPUT_FORMATS) { 157 List<Size> supportedSizes = extensionChars.getExtensionSupportedSizes( 158 config.getExtension(), format); 159 if (supportedSizes != null) { 160 supportedCaptureSizes.put(format, supportedSizes); 161 } 162 } 163 Surface burstCaptureSurface = CameraExtensionUtils.getBurstCaptureSurface( 164 config.getOutputConfigurations(), supportedCaptureSizes); 165 if (burstCaptureSurface != null) { 166 suitableSurfaceCount++; 167 } 168 169 if (suitableSurfaceCount != config.getOutputConfigurations().size()) { 170 throw new IllegalArgumentException("One or more unsupported output surfaces found!"); 171 } 172 173 IAdvancedExtenderImpl extender = CameraExtensionCharacteristics.initializeAdvancedExtension( 174 config.getExtension()); 175 extender.init(cameraId); 176 177 CameraAdvancedExtensionSessionImpl ret = new CameraAdvancedExtensionSessionImpl(clientId, 178 extender, cameraDevice, repeatingRequestSurface, burstCaptureSurface, 179 config.getStateCallback(), config.getExecutor(), sessionId); 180 ret.initialize(); 181 182 return ret; 183 } 184 CameraAdvancedExtensionSessionImpl(long extensionClientId, @NonNull IAdvancedExtenderImpl extender, @NonNull CameraDevice cameraDevice, @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, @NonNull CameraExtensionSession.StateCallback callback, @NonNull Executor executor, int sessionId)185 private CameraAdvancedExtensionSessionImpl(long extensionClientId, 186 @NonNull IAdvancedExtenderImpl extender, @NonNull CameraDevice cameraDevice, 187 @Nullable Surface repeatingRequestSurface, @Nullable Surface burstCaptureSurface, 188 @NonNull CameraExtensionSession.StateCallback callback, @NonNull Executor executor, 189 int sessionId) { 190 mExtensionClientId = extensionClientId; 191 mAdvancedExtender = extender; 192 mCameraDevice = cameraDevice; 193 mCallbacks = callback; 194 mExecutor = executor; 195 mClientRepeatingRequestSurface = repeatingRequestSurface; 196 mClientCaptureSurface = burstCaptureSurface; 197 mHandlerThread = new HandlerThread(TAG); 198 mHandlerThread.start(); 199 mHandler = new Handler(mHandlerThread.getLooper()); 200 mInitialized = false; 201 mInitializeHandler = new InitializeSessionHandler(); 202 mSessionId = sessionId; 203 } 204 205 /** 206 * @hide 207 */ initialize()208 public synchronized void initialize() throws CameraAccessException, RemoteException { 209 if (mInitialized) { 210 Log.d(TAG, "Session already initialized"); 211 return; 212 } 213 214 OutputSurface previewSurface = initializeParcelable(mClientRepeatingRequestSurface); 215 OutputSurface captureSurface = initializeParcelable(mClientCaptureSurface); 216 mSessionProcessor = mAdvancedExtender.getSessionProcessor(); 217 CameraSessionConfig sessionConfig = mSessionProcessor.initSession(mCameraDevice.getId(), 218 previewSurface, captureSurface); 219 List<CameraOutputConfig> outputConfigs = sessionConfig.outputConfigs; 220 ArrayList<OutputConfiguration> outputList = new ArrayList<>(); 221 for (CameraOutputConfig output : outputConfigs) { 222 Surface outputSurface = initializeSurfrace(output); 223 if (outputSurface == null) { 224 continue; 225 } 226 OutputConfiguration cameraOutput = new OutputConfiguration(output.surfaceGroupId, 227 outputSurface); 228 229 if (output.isMultiResolutionOutput) { 230 cameraOutput.setMultiResolutionOutput(); 231 } 232 if ((output.sharedSurfaceConfigs != null) && !output.sharedSurfaceConfigs.isEmpty()) { 233 cameraOutput.enableSurfaceSharing(); 234 for (CameraOutputConfig sharedOutputConfig : output.sharedSurfaceConfigs) { 235 Surface sharedSurface = initializeSurfrace(sharedOutputConfig); 236 if (sharedSurface == null) { 237 continue; 238 } 239 cameraOutput.addSurface(sharedSurface); 240 mCameraConfigMap.put(sharedSurface, sharedOutputConfig); 241 } 242 } 243 244 cameraOutput.setPhysicalCameraId(output.physicalCameraId); 245 outputList.add(cameraOutput); 246 mCameraConfigMap.put(cameraOutput.getSurface(), output); 247 } 248 249 SessionConfiguration sessionConfiguration = new SessionConfiguration( 250 SessionConfiguration.SESSION_REGULAR, outputList, 251 new CameraExtensionUtils.HandlerExecutor(mHandler), new SessionStateHandler()); 252 253 if ((sessionConfig.sessionParameter != null) && 254 (!sessionConfig.sessionParameter.isEmpty())) { 255 CaptureRequest.Builder requestBuilder = mCameraDevice.createCaptureRequest( 256 sessionConfig.sessionTemplateId); 257 CaptureRequest sessionRequest = requestBuilder.build(); 258 CameraMetadataNative.update(sessionRequest.getNativeMetadata(), 259 sessionConfig.sessionParameter); 260 sessionConfiguration.setSessionParameters(sessionRequest); 261 } 262 263 mCameraDevice.createCaptureSession(sessionConfiguration); 264 } 265 initializeParcelable(CaptureResult result)266 private static ParcelCaptureResult initializeParcelable(CaptureResult result) { 267 ParcelCaptureResult ret = new ParcelCaptureResult(); 268 ret.cameraId = result.getCameraId(); 269 ret.results = result.getNativeMetadata(); 270 ret.parent = result.getRequest(); 271 ret.sequenceId = result.getSequenceId(); 272 ret.frameNumber = result.getFrameNumber(); 273 274 return ret; 275 } 276 initializeParcelable(TotalCaptureResult totalResult)277 private static ParcelTotalCaptureResult initializeParcelable(TotalCaptureResult totalResult) { 278 ParcelTotalCaptureResult ret = new ParcelTotalCaptureResult(); 279 ret.logicalCameraId = totalResult.getCameraId(); 280 ret.results = totalResult.getNativeMetadata(); 281 ret.parent = totalResult.getRequest(); 282 ret.sequenceId = totalResult.getSequenceId(); 283 ret.frameNumber = totalResult.getFrameNumber(); 284 ret.sessionId = totalResult.getSessionId(); 285 ret.partials = new ArrayList<>(totalResult.getPartialResults().size()); 286 for (CaptureResult partial : totalResult.getPartialResults()) { 287 ret.partials.add(initializeParcelable(partial)); 288 } 289 Map<String, TotalCaptureResult> physicalResults = 290 totalResult.getPhysicalCameraTotalResults(); 291 ret.physicalResult = new ArrayList<>(physicalResults.size()); 292 for (TotalCaptureResult physicalResult : physicalResults.values()) { 293 ret.physicalResult.add(new PhysicalCaptureResultInfo(physicalResult.getCameraId(), 294 physicalResult.getNativeMetadata())); 295 } 296 297 return ret; 298 } 299 initializeParcelable(Surface s)300 private static OutputSurface initializeParcelable(Surface s) { 301 OutputSurface ret = new OutputSurface(); 302 if (s != null) { 303 ret.surface = s; 304 ret.size = new android.hardware.camera2.extension.Size(); 305 Size surfaceSize = SurfaceUtils.getSurfaceSize(s); 306 ret.size.width = surfaceSize.getWidth(); 307 ret.size.height = surfaceSize.getHeight(); 308 ret.imageFormat = SurfaceUtils.getSurfaceFormat(s); 309 } else { 310 ret.surface = null; 311 ret.size = new android.hardware.camera2.extension.Size(); 312 ret.size.width = -1; 313 ret.size.height = -1; 314 ret.imageFormat = ImageFormat.UNKNOWN; 315 } 316 317 return ret; 318 } 319 320 @Override getDevice()321 public @NonNull CameraDevice getDevice() { 322 synchronized (mInterfaceLock) { 323 return mCameraDevice; 324 } 325 } 326 327 @Override setRepeatingRequest(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)328 public int setRepeatingRequest(@NonNull CaptureRequest request, @NonNull Executor executor, 329 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 330 int seqId = -1; 331 synchronized (mInterfaceLock) { 332 if (!mInitialized) { 333 throw new IllegalStateException("Uninitialized component"); 334 } 335 336 if (mClientRepeatingRequestSurface == null) { 337 throw new IllegalArgumentException("No registered preview surface"); 338 } 339 340 if (!request.containsTarget(mClientRepeatingRequestSurface) || 341 (request.getTargets().size() != 1)) { 342 throw new IllegalArgumentException("Invalid repeating request output target!"); 343 } 344 345 try { 346 mSessionProcessor.setParameters(request); 347 348 seqId = mSessionProcessor.startRepeating(new RequestCallbackHandler(request, 349 executor, listener)); 350 } catch (RemoteException e) { 351 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 352 "Failed to enable repeating request, extension service failed to respond!"); 353 } 354 } 355 356 return seqId; 357 } 358 359 @Override capture(@onNull CaptureRequest request, @NonNull Executor executor, @NonNull ExtensionCaptureCallback listener)360 public int capture(@NonNull CaptureRequest request, 361 @NonNull Executor executor, 362 @NonNull ExtensionCaptureCallback listener) throws CameraAccessException { 363 int seqId = -1; 364 synchronized (mInterfaceLock) { 365 if (!mInitialized) { 366 throw new IllegalStateException("Uninitialized component"); 367 } 368 369 if (request.getTargets().size() != 1) { 370 throw new IllegalArgumentException("Single capture to both preview & still" + 371 " capture outputs target is not supported!"); 372 } 373 374 if ((mClientCaptureSurface != null) && request.containsTarget(mClientCaptureSurface)) { 375 try { 376 mSessionProcessor.setParameters(request); 377 378 seqId = mSessionProcessor.startCapture(new RequestCallbackHandler(request, 379 executor, listener)); 380 } catch (RemoteException e) { 381 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, "Failed " + 382 " to submit capture request, extension service failed to respond!"); 383 } 384 } else if ((mClientRepeatingRequestSurface != null) && 385 request.containsTarget(mClientRepeatingRequestSurface)) { 386 try { 387 seqId = mSessionProcessor.startTrigger(request, 388 new RequestCallbackHandler(request, executor, listener)); 389 } catch (RemoteException e) { 390 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, "Failed " + 391 " to submit trigger request, extension service failed to respond!"); 392 } 393 } else { 394 throw new IllegalArgumentException("Invalid single capture output target!"); 395 } 396 } 397 398 return seqId; 399 } 400 401 @Override stopRepeating()402 public void stopRepeating() throws CameraAccessException { 403 synchronized (mInterfaceLock) { 404 if (!mInitialized) { 405 throw new IllegalStateException("Uninitialized component"); 406 } 407 408 mCaptureSession.stopRepeating(); 409 410 try { 411 mSessionProcessor.stopRepeating(); 412 } catch (RemoteException e) { 413 throw new CameraAccessException(CameraAccessException.CAMERA_ERROR, 414 "Failed to notify about the end of repeating request, extension service" 415 + " failed to respond!"); 416 } 417 } 418 } 419 420 @Override close()421 public void close() throws CameraAccessException { 422 synchronized (mInterfaceLock) { 423 if (mInitialized) { 424 try { 425 mCaptureSession.stopRepeating(); 426 mSessionProcessor.stopRepeating(); 427 mSessionProcessor.onCaptureSessionEnd(); 428 } catch (RemoteException e) { 429 Log.e(TAG, "Failed to stop the repeating request or end the session," 430 + " , extension service does not respond!") ; 431 } 432 mCaptureSession.close(); 433 } 434 } 435 } 436 release(boolean skipCloseNotification)437 public void release(boolean skipCloseNotification) { 438 boolean notifyClose = false; 439 440 synchronized (mInterfaceLock) { 441 mHandlerThread.quitSafely(); 442 443 if (mSessionProcessor != null) { 444 try { 445 mSessionProcessor.deInitSession(); 446 } catch (RemoteException e) { 447 Log.e(TAG, "Failed to de-initialize session processor, extension service" 448 + " does not respond!") ; 449 } 450 mSessionProcessor = null; 451 } 452 453 if (mExtensionClientId >= 0) { 454 CameraExtensionCharacteristics.unregisterClient(mExtensionClientId); 455 if (mInitialized) { 456 notifyClose = true; 457 CameraExtensionCharacteristics.releaseSession(); 458 } 459 } 460 mInitialized = false; 461 462 for (ImageReader reader : mReaderMap.values()) { 463 reader.close(); 464 } 465 mReaderMap.clear(); 466 467 mClientRepeatingRequestSurface = null; 468 mClientCaptureSurface = null; 469 } 470 471 if (notifyClose && !skipCloseNotification) { 472 final long ident = Binder.clearCallingIdentity(); 473 try { 474 mExecutor.execute(() -> mCallbacks.onClosed( 475 CameraAdvancedExtensionSessionImpl.this)); 476 } finally { 477 Binder.restoreCallingIdentity(ident); 478 } 479 } 480 } 481 notifyConfigurationFailure()482 private void notifyConfigurationFailure() { 483 synchronized (mInterfaceLock) { 484 if (mInitialized) { 485 return; 486 } 487 } 488 489 release(true /*skipCloseNotification*/); 490 491 final long ident = Binder.clearCallingIdentity(); 492 try { 493 mExecutor.execute( 494 () -> mCallbacks.onConfigureFailed( 495 CameraAdvancedExtensionSessionImpl.this)); 496 } finally { 497 Binder.restoreCallingIdentity(ident); 498 } 499 } 500 501 private class SessionStateHandler extends 502 android.hardware.camera2.CameraCaptureSession.StateCallback { 503 @Override onClosed(@onNull CameraCaptureSession session)504 public void onClosed(@NonNull CameraCaptureSession session) { 505 release(false /*skipCloseNotification*/); 506 } 507 508 @Override onConfigureFailed(@onNull CameraCaptureSession session)509 public void onConfigureFailed(@NonNull CameraCaptureSession session) { 510 notifyConfigurationFailure(); 511 } 512 513 @Override onConfigured(@onNull CameraCaptureSession session)514 public void onConfigured(@NonNull CameraCaptureSession session) { 515 synchronized (mInterfaceLock) { 516 mCaptureSession = session; 517 try { 518 CameraExtensionCharacteristics.initializeSession(mInitializeHandler); 519 } catch (RemoteException e) { 520 Log.e(TAG, "Failed to initialize session! Extension service does" 521 + " not respond!"); 522 notifyConfigurationFailure(); 523 } 524 } 525 } 526 } 527 528 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 529 @Override onSuccess()530 public void onSuccess() { 531 boolean status = true; 532 synchronized (mInterfaceLock) { 533 try { 534 mSessionProcessor.onCaptureSessionStart(mRequestProcessor); 535 mInitialized = true; 536 } catch (RemoteException e) { 537 Log.e(TAG, "Failed to start capture session," 538 + " extension service does not respond!"); 539 status = false; 540 mCaptureSession.close(); 541 } 542 } 543 544 if (status) { 545 final long ident = Binder.clearCallingIdentity(); 546 try { 547 mExecutor.execute( 548 () -> mCallbacks.onConfigured(CameraAdvancedExtensionSessionImpl.this)); 549 } finally { 550 Binder.restoreCallingIdentity(ident); 551 } 552 } else { 553 notifyConfigurationFailure(); 554 } 555 } 556 557 @Override onFailure()558 public void onFailure() { 559 mCaptureSession.close(); 560 Log.e(TAG, "Failed to initialize proxy service session!" 561 + " This can happen when trying to configure multiple " 562 + "concurrent extension sessions!"); 563 notifyConfigurationFailure(); 564 } 565 } 566 567 private final class RequestCallbackHandler extends ICaptureCallback.Stub { 568 private final CaptureRequest mClientRequest; 569 private final Executor mClientExecutor; 570 private final ExtensionCaptureCallback mClientCallbacks; 571 RequestCallbackHandler(@onNull CaptureRequest clientRequest, @NonNull Executor clientExecutor, @NonNull ExtensionCaptureCallback clientCallbacks)572 private RequestCallbackHandler(@NonNull CaptureRequest clientRequest, 573 @NonNull Executor clientExecutor, 574 @NonNull ExtensionCaptureCallback clientCallbacks) { 575 mClientRequest = clientRequest; 576 mClientExecutor = clientExecutor; 577 mClientCallbacks = clientCallbacks; 578 } 579 580 @Override onCaptureStarted(int captureSequenceId, long timestamp)581 public void onCaptureStarted(int captureSequenceId, long timestamp) { 582 final long ident = Binder.clearCallingIdentity(); 583 try { 584 mClientExecutor.execute( 585 () -> mClientCallbacks.onCaptureStarted( 586 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 587 timestamp)); 588 } finally { 589 Binder.restoreCallingIdentity(ident); 590 } 591 } 592 593 @Override onCaptureProcessStarted(int captureSequenceId)594 public void onCaptureProcessStarted(int captureSequenceId) { 595 final long ident = Binder.clearCallingIdentity(); 596 try { 597 mClientExecutor.execute( 598 () -> mClientCallbacks.onCaptureProcessStarted( 599 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 600 } finally { 601 Binder.restoreCallingIdentity(ident); 602 } 603 } 604 605 @Override onCaptureFailed(int captureSequenceId)606 public void onCaptureFailed(int captureSequenceId) { 607 final long ident = Binder.clearCallingIdentity(); 608 try { 609 mClientExecutor.execute( 610 () -> mClientCallbacks.onCaptureFailed( 611 CameraAdvancedExtensionSessionImpl.this, mClientRequest)); 612 } finally { 613 Binder.restoreCallingIdentity(ident); 614 } 615 } 616 617 @Override onCaptureSequenceCompleted(int captureSequenceId)618 public void onCaptureSequenceCompleted(int captureSequenceId) { 619 final long ident = Binder.clearCallingIdentity(); 620 try { 621 mClientExecutor.execute( 622 () -> mClientCallbacks.onCaptureSequenceCompleted( 623 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 624 } finally { 625 Binder.restoreCallingIdentity(ident); 626 } 627 } 628 629 @Override onCaptureSequenceAborted(int captureSequenceId)630 public void onCaptureSequenceAborted(int captureSequenceId) { 631 final long ident = Binder.clearCallingIdentity(); 632 try { 633 mClientExecutor.execute( 634 () -> mClientCallbacks.onCaptureSequenceAborted( 635 CameraAdvancedExtensionSessionImpl.this, captureSequenceId)); 636 } finally { 637 Binder.restoreCallingIdentity(ident); 638 } 639 } 640 641 @Override onCaptureCompleted(long timestamp, int requestId, CameraMetadataNative result)642 public void onCaptureCompleted(long timestamp, int requestId, CameraMetadataNative result) { 643 if (result == null) { 644 Log.e(TAG,"Invalid capture result!"); 645 return; 646 } 647 648 result.set(CaptureResult.SENSOR_TIMESTAMP, timestamp); 649 TotalCaptureResult totalResult = new TotalCaptureResult(mCameraDevice.getId(), result, 650 mClientRequest, requestId, timestamp, new ArrayList<>(), mSessionId, 651 new PhysicalCaptureResultInfo[0]); 652 final long ident = Binder.clearCallingIdentity(); 653 try { 654 mExecutor.execute( 655 () -> mClientCallbacks.onCaptureResultAvailable( 656 CameraAdvancedExtensionSessionImpl.this, mClientRequest, 657 totalResult)); 658 } finally { 659 Binder.restoreCallingIdentity(ident); 660 } 661 } 662 } 663 664 private final class CaptureCallbackHandler extends CameraCaptureSession.CaptureCallback { 665 private final IRequestCallback mCallback; 666 CaptureCallbackHandler(IRequestCallback callback)667 public CaptureCallbackHandler(IRequestCallback callback) { 668 mCallback = callback; 669 } 670 671 @Override onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, Surface target, long frameNumber)672 public void onCaptureBufferLost(CameraCaptureSession session, CaptureRequest request, 673 Surface target, long frameNumber) { 674 try { 675 if (request.getTag() instanceof Integer) { 676 Integer requestId = (Integer) request.getTag(); 677 mCallback.onCaptureBufferLost(requestId, frameNumber, 678 mCameraConfigMap.get(target).outputId.id); 679 } else { 680 Log.e(TAG, "Invalid capture request tag!"); 681 } 682 } catch (RemoteException e) { 683 Log.e(TAG, "Failed to notify lost capture buffer, extension service doesn't" 684 + " respond!"); 685 } 686 } 687 688 @Override onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result)689 public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, 690 TotalCaptureResult result) { 691 try { 692 if (request.getTag() instanceof Integer) { 693 Integer requestId = (Integer) request.getTag(); 694 mCallback.onCaptureCompleted(requestId, initializeParcelable(result)); 695 } else { 696 Log.e(TAG, "Invalid capture request tag!"); 697 } 698 } catch (RemoteException e) { 699 Log.e(TAG, "Failed to notify capture result, extension service doesn't" 700 + " respond!"); 701 } 702 } 703 704 @Override onCaptureFailed(CameraCaptureSession session, CaptureRequest request, CaptureFailure failure)705 public void onCaptureFailed(CameraCaptureSession session, CaptureRequest request, 706 CaptureFailure failure) { 707 try { 708 if (request.getTag() instanceof Integer) { 709 Integer requestId = (Integer) request.getTag(); 710 android.hardware.camera2.extension.CaptureFailure captureFailure = 711 new android.hardware.camera2.extension.CaptureFailure(); 712 captureFailure.request = request; 713 captureFailure.reason = failure.getReason(); 714 captureFailure.errorPhysicalCameraId = failure.getPhysicalCameraId(); 715 captureFailure.frameNumber = failure.getFrameNumber(); 716 captureFailure.sequenceId = failure.getSequenceId(); 717 captureFailure.dropped = !failure.wasImageCaptured(); 718 mCallback.onCaptureFailed(requestId, captureFailure); 719 } else { 720 Log.e(TAG, "Invalid capture request tag!"); 721 } 722 } catch (RemoteException e) { 723 Log.e(TAG, "Failed to notify capture failure, extension service doesn't" 724 + " respond!"); 725 } 726 } 727 728 @Override onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, CaptureResult partialResult)729 public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, 730 CaptureResult partialResult) { 731 try { 732 if (request.getTag() instanceof Integer) { 733 Integer requestId = (Integer) request.getTag(); 734 mCallback.onCaptureProgressed(requestId, initializeParcelable(partialResult)); 735 } else { 736 Log.e(TAG, "Invalid capture request tag!"); 737 } 738 } catch (RemoteException e) { 739 Log.e(TAG, "Failed to notify capture partial result, extension service doesn't" 740 + " respond!"); 741 } 742 } 743 744 @Override onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId)745 public void onCaptureSequenceAborted(CameraCaptureSession session, int sequenceId) { 746 try { 747 mCallback.onCaptureSequenceAborted(sequenceId); 748 } catch (RemoteException e) { 749 Log.e(TAG, "Failed to notify aborted sequence, extension service doesn't" 750 + " respond!"); 751 } 752 } 753 754 @Override onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, long frameNumber)755 public void onCaptureSequenceCompleted(CameraCaptureSession session, int sequenceId, 756 long frameNumber) { 757 try { 758 mCallback.onCaptureSequenceCompleted(sequenceId, frameNumber); 759 } catch (RemoteException e) { 760 Log.e(TAG, "Failed to notify sequence complete, extension service doesn't" 761 + " respond!"); 762 } 763 } 764 765 @Override onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber)766 public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, 767 long timestamp, long frameNumber) { 768 try { 769 if (request.getTag() instanceof Integer) { 770 Integer requestId = (Integer) request.getTag(); 771 mCallback.onCaptureStarted(requestId, frameNumber, timestamp); 772 } else { 773 Log.e(TAG, "Invalid capture request tag!"); 774 } 775 } catch (RemoteException e) { 776 Log.e(TAG, "Failed to notify capture started, extension service doesn't" 777 + " respond!"); 778 } 779 } 780 } 781 782 private static final class ImageReaderHandler implements ImageReader.OnImageAvailableListener { 783 private final OutputConfigId mOutputConfigId; 784 private final IImageProcessorImpl mIImageProcessor; 785 private final String mPhysicalCameraId; 786 ImageReaderHandler(int outputConfigId, IImageProcessorImpl iImageProcessor, String physicalCameraId)787 private ImageReaderHandler(int outputConfigId, 788 IImageProcessorImpl iImageProcessor, String physicalCameraId) { 789 mOutputConfigId = new OutputConfigId(); 790 mOutputConfigId.id = outputConfigId; 791 mIImageProcessor = iImageProcessor; 792 mPhysicalCameraId = physicalCameraId; 793 } 794 795 @Override onImageAvailable(ImageReader reader)796 public void onImageAvailable(ImageReader reader) { 797 if (mIImageProcessor == null) { 798 return; 799 } 800 801 Image img; 802 try { 803 img = reader.acquireNextImage(); 804 } catch (IllegalStateException e) { 805 Log.e(TAG, "Failed to acquire image, too many images pending!"); 806 return; 807 } 808 if (img == null) { 809 Log.e(TAG, "Invalid image!"); 810 return; 811 } 812 813 try { 814 reader.detachImage(img); 815 } catch(Exception e) { 816 Log.e(TAG, "Failed to detach image"); 817 img.close(); 818 return; 819 } 820 821 ParcelImage parcelImage = new ParcelImage(); 822 parcelImage.buffer = img.getHardwareBuffer(); 823 try { 824 SyncFence fd = img.getFence(); 825 if (fd.isValid()) { 826 parcelImage.fence = fd.getFdDup(); 827 } 828 } catch (IOException e) { 829 Log.e(TAG, "Failed to parcel buffer fence!"); 830 } 831 parcelImage.width = img.getWidth(); 832 parcelImage.height = img.getHeight(); 833 parcelImage.format = img.getFormat(); 834 parcelImage.timestamp = img.getTimestamp(); 835 parcelImage.transform = img.getTransform(); 836 parcelImage.scalingMode = img.getScalingMode(); 837 parcelImage.planeCount = img.getPlaneCount(); 838 parcelImage.crop = img.getCropRect(); 839 840 try { 841 mIImageProcessor.onNextImageAvailable(mOutputConfigId, parcelImage, 842 mPhysicalCameraId); 843 } catch (RemoteException e) { 844 Log.e(TAG, "Failed to propagate image buffer on output surface id: " + 845 mOutputConfigId + " extension service does not respond!"); 846 } finally { 847 parcelImage.buffer.close(); 848 img.close(); 849 } 850 } 851 } 852 853 private final class RequestProcessor extends IRequestProcessorImpl.Stub { 854 @Override setImageProcessor(OutputConfigId outputConfigId, IImageProcessorImpl imageProcessor)855 public void setImageProcessor(OutputConfigId outputConfigId, 856 IImageProcessorImpl imageProcessor) { 857 synchronized (mInterfaceLock) { 858 if (mReaderMap.containsKey(outputConfigId.id)) { 859 ImageReader reader = mReaderMap.get(outputConfigId.id); 860 String physicalCameraId = null; 861 if (mCameraConfigMap.containsKey(reader.getSurface())) { 862 physicalCameraId = 863 mCameraConfigMap.get(reader.getSurface()).physicalCameraId; 864 reader.setOnImageAvailableListener(new ImageReaderHandler(outputConfigId.id, 865 imageProcessor, physicalCameraId), mHandler); 866 } else { 867 Log.e(TAG, "Camera output configuration for ImageReader with " + 868 " config Id " + outputConfigId.id + " not found!"); 869 } 870 } else { 871 Log.e(TAG, "ImageReader with output config id: " + outputConfigId.id + 872 " not found!"); 873 } 874 } 875 } 876 877 @Override submit(Request request, IRequestCallback callback)878 public int submit(Request request, IRequestCallback callback) { 879 ArrayList<Request> captureList = new ArrayList<>(); 880 captureList.add(request); 881 return submitBurst(captureList, callback); 882 } 883 884 @Override submitBurst(List<Request> requests, IRequestCallback callback)885 public int submitBurst(List<Request> requests, IRequestCallback callback) { 886 int seqId = -1; 887 try { 888 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 889 ArrayList<CaptureRequest> captureRequests = new ArrayList<>(); 890 for (Request request : requests) { 891 captureRequests.add(initializeCaptureRequest(mCameraDevice, request, 892 mCameraConfigMap)); 893 } 894 seqId = mCaptureSession.captureBurstRequests(captureRequests, 895 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 896 } catch (CameraAccessException e) { 897 Log.e(TAG, "Failed to submit capture requests!"); 898 } catch (IllegalStateException e) { 899 Log.e(TAG, "Capture session closed!"); 900 } 901 902 return seqId; 903 } 904 905 @Override setRepeating(Request request, IRequestCallback callback)906 public int setRepeating(Request request, IRequestCallback callback) { 907 int seqId = -1; 908 try { 909 CaptureRequest repeatingRequest = initializeCaptureRequest(mCameraDevice, 910 request, mCameraConfigMap); 911 CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback); 912 seqId = mCaptureSession.setSingleRepeatingRequest(repeatingRequest, 913 new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback); 914 } catch (CameraAccessException e) { 915 Log.e(TAG, "Failed to enable repeating request!"); 916 } catch (IllegalStateException e) { 917 Log.e(TAG, "Capture session closed!"); 918 } 919 920 return seqId; 921 } 922 923 @Override abortCaptures()924 public void abortCaptures() { 925 try { 926 mCaptureSession.abortCaptures(); 927 } catch (CameraAccessException e) { 928 Log.e(TAG, "Failed during capture abort!"); 929 } catch (IllegalStateException e) { 930 Log.e(TAG, "Capture session closed!"); 931 } 932 } 933 934 @Override stopRepeating()935 public void stopRepeating() { 936 try { 937 mCaptureSession.stopRepeating(); 938 } catch (CameraAccessException e) { 939 Log.e(TAG, "Failed during repeating capture stop!"); 940 } catch (IllegalStateException e) { 941 Log.e(TAG, "Capture session closed!"); 942 } 943 } 944 } 945 initializeCaptureRequest(CameraDevice cameraDevice, Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap)946 private static CaptureRequest initializeCaptureRequest(CameraDevice cameraDevice, 947 Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap) 948 throws CameraAccessException { 949 CaptureRequest.Builder builder = cameraDevice.createCaptureRequest(request.templateId); 950 for (OutputConfigId configId : request.targetOutputConfigIds) { 951 boolean found = false; 952 for (Map.Entry<Surface, CameraOutputConfig> entry : surfaceIdMap.entrySet()) { 953 if (entry.getValue().outputId.id == configId.id) { 954 builder.addTarget(entry.getKey()); 955 found = true; 956 break; 957 } 958 } 959 960 if (!found) { 961 Log.e(TAG, "Surface with output id: " + configId.id + 962 " not found among registered camera outputs!"); 963 } 964 } 965 966 builder.setTag(request.requestId); 967 CaptureRequest ret = builder.build(); 968 CameraMetadataNative.update(ret.getNativeMetadata(), request.parameters); 969 return ret; 970 } 971 initializeSurfrace(CameraOutputConfig output)972 private Surface initializeSurfrace(CameraOutputConfig output) { 973 switch(output.type) { 974 case CameraOutputConfig.TYPE_SURFACE: 975 if (output.surface == null) { 976 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 977 ", skipping!"); 978 return null; 979 } 980 return output.surface; 981 case CameraOutputConfig.TYPE_IMAGEREADER: 982 if ((output.imageFormat == ImageFormat.UNKNOWN) || (output.size.width <= 0) || 983 (output.size.height <= 0)) { 984 Log.w(TAG, "Unsupported client output id: " + output.outputId.id + 985 ", skipping!"); 986 return null; 987 } 988 ImageReader reader = ImageReader.newInstance(output.size.width, 989 output.size.height, output.imageFormat, output.capacity); 990 mReaderMap.put(output.outputId.id, reader); 991 return reader.getSurface(); 992 case CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER: 993 // Support for multi-resolution outputs to be added in future releases 994 default: 995 throw new IllegalArgumentException("Unsupported output config type: " + 996 output.type); 997 } 998 } 999 } 1000