1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.hardware.camera2; 17 18 import android.annotation.FlaggedApi; 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.annotation.SuppressLint; 23 import android.content.ComponentName; 24 import android.content.ContentResolver; 25 import android.content.Context; 26 import android.content.Intent; 27 import android.content.pm.PackageManager; 28 import android.content.ServiceConnection; 29 import android.graphics.ImageFormat; 30 import android.hardware.camera2.CameraCharacteristics.Key; 31 import android.hardware.camera2.extension.IAdvancedExtenderImpl; 32 import android.hardware.camera2.extension.ICameraExtensionsProxyService; 33 import android.hardware.camera2.extension.IImageCaptureExtenderImpl; 34 import android.hardware.camera2.extension.IInitializeSessionCallback; 35 import android.hardware.camera2.extension.IPreviewExtenderImpl; 36 import android.hardware.camera2.extension.LatencyRange; 37 import android.hardware.camera2.extension.SizeList; 38 import android.hardware.camera2.impl.CameraExtensionUtils; 39 import android.hardware.camera2.impl.CameraMetadataNative; 40 import android.hardware.camera2.impl.ExtensionKey; 41 import android.hardware.camera2.impl.PublicKey; 42 import android.hardware.camera2.params.ExtensionSessionConfiguration; 43 import android.hardware.camera2.params.StreamConfigurationMap; 44 import android.os.Binder; 45 import android.os.ConditionVariable; 46 import android.os.IBinder; 47 import android.os.RemoteException; 48 import android.os.SystemProperties; 49 import android.provider.Settings; 50 import android.util.IntArray; 51 import android.util.Log; 52 import android.util.Pair; 53 import android.util.Range; 54 import android.util.Size; 55 56 import com.android.internal.camera.flags.Flags; 57 58 import java.lang.annotation.Retention; 59 import java.lang.annotation.RetentionPolicy; 60 import java.util.ArrayList; 61 import java.util.Arrays; 62 import java.util.Collections; 63 import java.util.HashMap; 64 import java.util.HashSet; 65 import java.util.List; 66 import java.util.Map; 67 import java.util.Objects; 68 import java.util.Set; 69 import java.util.concurrent.Future; 70 import java.util.concurrent.TimeUnit; 71 import java.util.concurrent.TimeoutException; 72 73 /** 74 * <p>Allows clients to query availability and supported resolutions of camera extensions.</p> 75 * 76 * <p>Camera extensions give camera clients access to device-specific algorithms and sequences that 77 * can improve the overall image quality of snapshots in various cases such as low light, selfies, 78 * portraits, and scenes that can benefit from enhanced dynamic range. Often such sophisticated 79 * processing sequences will rely on multiple camera frames as input and will produce a single 80 * output.</p> 81 * 82 * <p>Camera extensions are not guaranteed to be present on all devices so camera clients must 83 * query for their availability via {@link CameraExtensionCharacteristics#getSupportedExtensions()}. 84 * </p> 85 * 86 * <p>In order to use any available camera extension, camera clients must create a corresponding 87 * {@link CameraExtensionSession} via 88 * {@link CameraDevice#createExtensionSession(ExtensionSessionConfiguration)}</p> 89 * 90 * <p>Camera clients must be aware that device-specific camera extensions may support only a 91 * subset of the available camera resolutions and must first query 92 * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, int)} for supported 93 * single high-quality request output sizes and 94 * {@link CameraExtensionCharacteristics#getExtensionSupportedSizes(int, Class)} for supported 95 * repeating request output sizes.</p> 96 * 97 * <p>The extension characteristics for a given device are expected to remain static under 98 * normal operating conditions.</p> 99 * 100 * @see CameraManager#getCameraExtensionCharacteristics(String) 101 */ 102 public final class CameraExtensionCharacteristics { 103 private static final String TAG = "CameraExtensionCharacteristics"; 104 105 /** 106 * Device-specific extension implementation for automatic selection of particular extension 107 * such as HDR or NIGHT depending on the current lighting and environment conditions. 108 */ 109 public static final int EXTENSION_AUTOMATIC = 0; 110 111 /** 112 * Device-specific extension implementation which tends to smooth the skin and apply other 113 * cosmetic effects to people's faces. 114 */ 115 public static final int EXTENSION_FACE_RETOUCH = 1; 116 117 /** 118 * Device-specific extension implementation which tends to smooth the skin and apply other 119 * cosmetic effects to people's faces. 120 * 121 * @deprecated Use {@link #EXTENSION_FACE_RETOUCH} instead. 122 */ 123 public @Deprecated static final int EXTENSION_BEAUTY = EXTENSION_FACE_RETOUCH; 124 125 /** 126 * Device-specific extension implementation which can blur certain regions of the final image 127 * thereby "enhancing" focus for all remaining non-blurred parts. 128 */ 129 public static final int EXTENSION_BOKEH = 2; 130 131 /** 132 * Device-specific extension implementation for enhancing the dynamic range of the 133 * final image. 134 */ 135 public static final int EXTENSION_HDR = 3; 136 137 /** 138 * Device-specific extension implementation that aims to suppress noise and improve the 139 * overall image quality under low light conditions. 140 */ 141 public static final int EXTENSION_NIGHT = 4; 142 143 /** 144 * @hide 145 */ 146 @Retention(RetentionPolicy.SOURCE) 147 @IntDef(flag = true, value = {EXTENSION_AUTOMATIC, 148 EXTENSION_FACE_RETOUCH, 149 EXTENSION_BOKEH, 150 EXTENSION_HDR, 151 EXTENSION_NIGHT}) 152 public @interface Extension { 153 } 154 155 /** 156 * Default camera output in case additional processing from CameraX extensions is not needed 157 * 158 * @hide 159 */ 160 public static final int NON_PROCESSING_INPUT_FORMAT = ImageFormat.PRIVATE; 161 162 /** 163 * CameraX extensions require YUV_420_888 as default input for processing at the moment 164 * 165 * @hide 166 */ 167 public static final int PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888; 168 169 private static final @Extension 170 int[] EXTENSION_LIST = new int[]{ 171 EXTENSION_AUTOMATIC, 172 EXTENSION_FACE_RETOUCH, 173 EXTENSION_BOKEH, 174 EXTENSION_HDR, 175 EXTENSION_NIGHT}; 176 177 /** 178 * List of synthetic CameraCharacteristics keys that are supported in the extensions. 179 */ 180 private static final List<CameraCharacteristics.Key> 181 SUPPORTED_SYNTHETIC_CAMERA_CHARACTERISTICS = 182 Arrays.asList( 183 CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES, 184 CameraCharacteristics.REQUEST_AVAILABLE_COLOR_SPACE_PROFILES 185 ); 186 187 private final Context mContext; 188 private final String mCameraId; 189 private final Map<String, CameraCharacteristics> mCharacteristicsMap; 190 private final Map<String, CameraMetadataNative> mCharacteristicsMapNative; 191 192 /** 193 * @hide 194 */ CameraExtensionCharacteristics(Context context, String cameraId, Map<String, CameraCharacteristics> characteristicsMap)195 public CameraExtensionCharacteristics(Context context, String cameraId, 196 Map<String, CameraCharacteristics> characteristicsMap) { 197 mContext = context; 198 mCameraId = cameraId; 199 mCharacteristicsMap = characteristicsMap; 200 mCharacteristicsMapNative = 201 CameraExtensionUtils.getCharacteristicsMapNative(characteristicsMap); 202 } 203 getSupportedSizes(List<SizeList> sizesList, Integer format)204 private static ArrayList<Size> getSupportedSizes(List<SizeList> sizesList, 205 Integer format) { 206 ArrayList<Size> ret = new ArrayList<>(); 207 if ((sizesList != null) && (!sizesList.isEmpty())) { 208 for (SizeList entry : sizesList) { 209 if ((entry.format == format) && !entry.sizes.isEmpty()) { 210 for (android.hardware.camera2.extension.Size sz : entry.sizes) { 211 ret.add(new Size(sz.width, sz.height)); 212 } 213 return ret; 214 } 215 } 216 } 217 218 return ret; 219 } 220 generateSupportedSizes(List<SizeList> sizesList, Integer format, StreamConfigurationMap streamMap)221 private static List<Size> generateSupportedSizes(List<SizeList> sizesList, 222 Integer format, 223 StreamConfigurationMap streamMap) { 224 ArrayList<Size> ret = getSupportedSizes(sizesList, format); 225 226 if (format == ImageFormat.JPEG || format == ImageFormat.YUV_420_888 || 227 format == ImageFormat.PRIVATE) { 228 // Per API contract it is assumed that the extension is able to support all 229 // camera advertised sizes for JPEG, YUV_420_888 and PRIVATE in case it doesn't return 230 // a valid non-empty size list. 231 Size[] supportedSizes = streamMap.getOutputSizes(format); 232 if ((ret.isEmpty()) && (supportedSizes != null)) { 233 ret.addAll(Arrays.asList(supportedSizes)); 234 } 235 } 236 237 return ret; 238 } 239 generateJpegSupportedSizes(List<SizeList> sizesList, StreamConfigurationMap streamMap)240 private static List<Size> generateJpegSupportedSizes(List<SizeList> sizesList, 241 StreamConfigurationMap streamMap) { 242 ArrayList<Size> extensionSizes = getSupportedSizes(sizesList, ImageFormat.YUV_420_888); 243 HashSet<Size> supportedSizes = extensionSizes.isEmpty() ? new HashSet<>(Arrays.asList( 244 streamMap.getOutputSizes(ImageFormat.YUV_420_888))) : new HashSet<>(extensionSizes); 245 HashSet<Size> supportedJpegSizes = new HashSet<>(Arrays.asList(streamMap.getOutputSizes( 246 ImageFormat.JPEG))); 247 supportedSizes.retainAll(supportedJpegSizes); 248 249 return new ArrayList<>(supportedSizes); 250 } 251 252 /** 253 * A per-process global camera extension manager instance, to track and 254 * initialize/release extensions depending on client activity. 255 */ 256 private static final class CameraExtensionManagerGlobal { 257 private static final String TAG = "CameraExtensionManagerGlobal"; 258 private static final String PROXY_PACKAGE_NAME = "com.android.cameraextensions"; 259 private static final String PROXY_SERVICE_NAME = 260 "com.android.cameraextensions.CameraExtensionsProxyService"; 261 262 private static final int FALLBACK_PACKAGE_NAME = 263 com.android.internal.R.string.config_extensionFallbackPackageName; 264 private static final int FALLBACK_SERVICE_NAME = 265 com.android.internal.R.string.config_extensionFallbackServiceName; 266 267 // Singleton instance 268 private static final CameraExtensionManagerGlobal GLOBAL_CAMERA_MANAGER = 269 new CameraExtensionManagerGlobal(); 270 private final Object mLock = new Object(); 271 private final int PROXY_SERVICE_DELAY_MS = 2000; 272 private ExtensionConnectionManager mConnectionManager = new ExtensionConnectionManager(); 273 private boolean mPermissionForFallbackEnabled = false; 274 private boolean mIsFallbackEnabled = false; 275 276 // Singleton, don't allow construction CameraExtensionManagerGlobal()277 private CameraExtensionManagerGlobal() {} 278 get()279 public static CameraExtensionManagerGlobal get() { 280 return GLOBAL_CAMERA_MANAGER; 281 } 282 releaseProxyConnectionLocked(Context ctx, int extension)283 private void releaseProxyConnectionLocked(Context ctx, int extension) { 284 if (mConnectionManager.getConnection(extension) != null) { 285 ctx.unbindService(mConnectionManager.getConnection(extension)); 286 mConnectionManager.setConnection(extension, null); 287 mConnectionManager.setProxy(extension, null); 288 mConnectionManager.resetConnectionCount(extension); 289 } 290 } 291 connectToProxyLocked(Context ctx, int extension, boolean useFallback)292 private void connectToProxyLocked(Context ctx, int extension, boolean useFallback) { 293 if (mConnectionManager.getConnection(extension) == null) { 294 Intent intent = new Intent(); 295 intent.setClassName(PROXY_PACKAGE_NAME, PROXY_SERVICE_NAME); 296 String vendorProxyPackage = SystemProperties.get( 297 "ro.vendor.camera.extensions.package"); 298 String vendorProxyService = SystemProperties.get( 299 "ro.vendor.camera.extensions.service"); 300 if (!vendorProxyPackage.isEmpty() && !vendorProxyService.isEmpty()) { 301 Log.v(TAG, 302 "Choosing the vendor camera extensions proxy package: " 303 + vendorProxyPackage); 304 Log.v(TAG, 305 "Choosing the vendor camera extensions proxy service: " 306 + vendorProxyService); 307 intent.setClassName(vendorProxyPackage, vendorProxyService); 308 } 309 310 if (useFallback) { 311 String packageName = ctx.getResources().getString(FALLBACK_PACKAGE_NAME); 312 String serviceName = ctx.getResources().getString(FALLBACK_SERVICE_NAME); 313 314 if (!packageName.isEmpty() && !serviceName.isEmpty()) { 315 Log.v(TAG, 316 "Choosing the fallback software implementation package: " 317 + packageName); 318 Log.v(TAG, 319 "Choosing the fallback software implementation service: " 320 + serviceName); 321 intent.setClassName(packageName, serviceName); 322 mIsFallbackEnabled = true; 323 } 324 } 325 326 InitializerFuture initFuture = new InitializerFuture(); 327 ServiceConnection connection = new ServiceConnection() { 328 @Override 329 public void onServiceDisconnected(ComponentName component) { 330 mConnectionManager.setConnection(extension, null); 331 mConnectionManager.setProxy(extension, null); 332 } 333 334 @Override 335 public void onServiceConnected(ComponentName component, IBinder binder) { 336 ICameraExtensionsProxyService proxy = 337 ICameraExtensionsProxyService.Stub.asInterface(binder); 338 mConnectionManager.setProxy(extension, proxy); 339 if (mConnectionManager.getProxy(extension) == null) { 340 throw new IllegalStateException("Camera Proxy service is null"); 341 } 342 try { 343 mConnectionManager.setAdvancedExtensionsSupported(extension, 344 mConnectionManager.getProxy(extension) 345 .advancedExtensionsSupported()); 346 } catch (RemoteException e) { 347 Log.e(TAG, "Remote IPC failed!"); 348 } 349 initFuture.setStatus(true); 350 } 351 }; 352 ctx.bindService(intent, Context.BIND_AUTO_CREATE | Context.BIND_IMPORTANT | 353 Context.BIND_ABOVE_CLIENT | Context.BIND_NOT_VISIBLE, 354 android.os.AsyncTask.THREAD_POOL_EXECUTOR, connection); 355 mConnectionManager.setConnection(extension, connection); 356 357 try { 358 initFuture.get(PROXY_SERVICE_DELAY_MS, TimeUnit.MILLISECONDS); 359 } catch (TimeoutException e) { 360 Log.e(TAG, "Timed out while initializing proxy service!"); 361 } 362 } 363 } 364 365 private static class InitializerFuture implements Future<Boolean> { 366 private volatile Boolean mStatus; 367 ConditionVariable mCondVar = new ConditionVariable(/*opened*/false); 368 setStatus(boolean status)369 public void setStatus(boolean status) { 370 mStatus = status; 371 mCondVar.open(); 372 } 373 374 @Override cancel(boolean mayInterruptIfRunning)375 public boolean cancel(boolean mayInterruptIfRunning) { 376 return false; // don't allow canceling this task 377 } 378 379 @Override isCancelled()380 public boolean isCancelled() { 381 return false; // can never cancel this task 382 } 383 384 @Override isDone()385 public boolean isDone() { 386 return mStatus != null; 387 } 388 389 @Override get()390 public Boolean get() { 391 mCondVar.block(); 392 return mStatus; 393 } 394 395 @Override get(long timeout, TimeUnit unit)396 public Boolean get(long timeout, TimeUnit unit) throws TimeoutException { 397 long timeoutMs = unit.convert(timeout, TimeUnit.MILLISECONDS); 398 if (!mCondVar.block(timeoutMs)) { 399 throw new TimeoutException( 400 "Failed to receive status after " + timeout + " " + unit); 401 } 402 403 if (mStatus == null) { 404 throw new AssertionError(); 405 } 406 return mStatus; 407 } 408 } 409 registerClientHelper(Context ctx, IBinder token, int extension, boolean useFallback)410 public boolean registerClientHelper(Context ctx, IBinder token, int extension, 411 boolean useFallback) { 412 synchronized (mLock) { 413 boolean ret = false; 414 connectToProxyLocked(ctx, extension, useFallback); 415 if (mConnectionManager.getProxy(extension) == null) { 416 return false; 417 } 418 mConnectionManager.incrementConnectionCount(extension); 419 420 try { 421 ret = mConnectionManager.getProxy(extension).registerClient(token); 422 } catch (RemoteException e) { 423 Log.e(TAG, "Failed to initialize extension! Extension service does " 424 + " not respond!"); 425 } 426 if (!ret) { 427 mConnectionManager.decrementConnectionCount(extension); 428 } 429 430 if (mConnectionManager.getConnectionCount(extension) <= 0) { 431 releaseProxyConnectionLocked(ctx, extension); 432 } 433 434 if (ret && useFallback && mIsFallbackEnabled) { 435 try { 436 InitializeSessionHandler cb = new InitializeSessionHandler(ctx); 437 initializeSession(cb, extension); 438 ret = mPermissionForFallbackEnabled; 439 } catch (RemoteException e) { 440 Log.e(TAG, "Failed to initialize extension. Extension service does not" 441 + " respond!"); 442 ret = false; 443 } finally { 444 releaseSession(extension); 445 } 446 } 447 448 return ret; 449 } 450 } 451 452 @SuppressLint("NonUserGetterCalled") registerClient(Context ctx, IBinder token, int extension, String cameraId, Map<String, CameraMetadataNative> characteristicsMapNative)453 public boolean registerClient(Context ctx, IBinder token, int extension, 454 String cameraId, Map<String, CameraMetadataNative> characteristicsMapNative) { 455 if (!SystemProperties.getBoolean("ro.camerax.extensions.enabled", 456 /*default*/ false)) { 457 Log.v(TAG, "Disabled camera extension property!"); 458 return false; 459 } 460 461 boolean ret = registerClientHelper(ctx, token, extension, false /*useFallback*/); 462 463 // Check if user enabled fallback impl 464 ContentResolver resolver = ctx.getContentResolver(); 465 int userEnabled = Settings.Secure.getInt(resolver, 466 Settings.Secure.CAMERA_EXTENSIONS_FALLBACK, 1); 467 468 boolean vendorImpl = true; 469 if (ret && (mConnectionManager.getProxy(extension) != null) && (userEnabled == 1)) { 470 // At this point, we are connected to either CameraExtensionsProxyService or 471 // the vendor extension proxy service. If the vendor does not support the 472 // extension, unregisterClient and re-register client with the proxy service 473 // containing the fallback impl 474 vendorImpl = isExtensionSupported(cameraId, extension, 475 characteristicsMapNative); 476 } 477 478 if (!vendorImpl) { 479 unregisterClient(ctx, token, extension); 480 ret = registerClientHelper(ctx, token, extension, true /*useFallback*/); 481 } 482 483 return ret; 484 } 485 unregisterClient(Context ctx, IBinder token, int extension)486 public void unregisterClient(Context ctx, IBinder token, int extension) { 487 synchronized (mLock) { 488 if (mConnectionManager.getProxy(extension) != null) { 489 try { 490 mConnectionManager.getProxy(extension).unregisterClient(token); 491 } catch (RemoteException e) { 492 Log.e(TAG, "Failed to de-initialize extension! Extension service does" 493 + " not respond!"); 494 } finally { 495 mConnectionManager.decrementConnectionCount(extension); 496 if (mConnectionManager.getConnectionCount(extension) <= 0) { 497 releaseProxyConnectionLocked(ctx, extension); 498 } 499 } 500 } 501 } 502 } 503 initializeSession(IInitializeSessionCallback cb, int extension)504 public void initializeSession(IInitializeSessionCallback cb, int extension) 505 throws RemoteException { 506 synchronized (mLock) { 507 if (mConnectionManager.getProxy(extension) != null 508 && !mConnectionManager.isSessionInitialized()) { 509 mConnectionManager.getProxy(extension).initializeSession(cb); 510 mConnectionManager.setSessionInitialized(true); 511 } else { 512 cb.onFailure(); 513 } 514 } 515 } 516 releaseSession(int extension)517 public void releaseSession(int extension) { 518 synchronized (mLock) { 519 if (mConnectionManager.getProxy(extension) != null) { 520 try { 521 mConnectionManager.getProxy(extension).releaseSession(); 522 mConnectionManager.setSessionInitialized(false); 523 mPermissionForFallbackEnabled = false; // Reset permission status 524 } catch (RemoteException e) { 525 Log.e(TAG, "Failed to release session! Extension service does" 526 + " not respond!"); 527 } 528 } 529 } 530 } 531 areAdvancedExtensionsSupported(int extension)532 public boolean areAdvancedExtensionsSupported(int extension) { 533 return mConnectionManager.areAdvancedExtensionsSupported(extension); 534 } 535 initializePreviewExtension(int extension)536 public IPreviewExtenderImpl initializePreviewExtension(int extension) 537 throws RemoteException { 538 synchronized (mLock) { 539 if (mConnectionManager.getProxy(extension) != null) { 540 return mConnectionManager.getProxy(extension) 541 .initializePreviewExtension(extension); 542 } else { 543 return null; 544 } 545 } 546 } 547 initializeImageExtension(int extension)548 public IImageCaptureExtenderImpl initializeImageExtension(int extension) 549 throws RemoteException { 550 synchronized (mLock) { 551 if (mConnectionManager.getProxy(extension) != null) { 552 return mConnectionManager.getProxy(extension) 553 .initializeImageExtension(extension); 554 } else { 555 return null; 556 } 557 } 558 } 559 initializeAdvancedExtension(int extension)560 public IAdvancedExtenderImpl initializeAdvancedExtension(int extension) 561 throws RemoteException { 562 synchronized (mLock) { 563 if (mConnectionManager.getProxy(extension) != null) { 564 return mConnectionManager.getProxy(extension) 565 .initializeAdvancedExtension(extension); 566 } else { 567 return null; 568 } 569 } 570 } 571 572 private class InitializeSessionHandler extends IInitializeSessionCallback.Stub { 573 private Context mContext; 574 InitializeSessionHandler(Context context)575 public InitializeSessionHandler(Context context) { 576 mContext = context; 577 } 578 579 @Override onSuccess()580 public void onSuccess() { 581 // Verify that the camera permission is granted if using 582 // the fallback implementation for an extension 583 String[] callingUidPackages = mContext.getPackageManager() 584 .getPackagesForUid(Binder.getCallingUid()); 585 String fallbackPackageName = mContext.getResources() 586 .getString(FALLBACK_PACKAGE_NAME); 587 588 if (!fallbackPackageName.isEmpty() 589 && Arrays.stream(callingUidPackages) 590 .anyMatch(fallbackPackageName::equals)) { 591 String[] cameraPermissions = { 592 android.Manifest.permission.SYSTEM_CAMERA, 593 android.Manifest.permission.CAMERA 594 }; 595 596 boolean allPermissionsGranted = true; 597 for (String permission : cameraPermissions) { 598 int permissionResult = mContext.checkPermission(permission, 599 Binder.getCallingPid(), Binder.getCallingUid()); 600 if (permissionResult != PackageManager.PERMISSION_GRANTED) { 601 Log.w(TAG, permission + " permission not granted for " 602 + fallbackPackageName + ", permission check result: " 603 + permissionResult); 604 allPermissionsGranted = false; 605 } 606 } 607 608 mPermissionForFallbackEnabled = allPermissionsGranted; 609 } 610 } 611 612 @Override onFailure()613 public void onFailure() { 614 Log.e(TAG, "Failed to initialize proxy service session!"); 615 } 616 } 617 618 private class ExtensionConnectionManager { 619 // Maps extension to ExtensionConnection 620 private Map<Integer, ExtensionConnection> mConnections = new HashMap<>(); 621 private boolean mSessionInitialized = false; 622 ExtensionConnectionManager()623 public ExtensionConnectionManager() { 624 IntArray extensionList = new IntArray(EXTENSION_LIST.length); 625 extensionList.addAll(EXTENSION_LIST); 626 627 for (int extensionType : extensionList.toArray()) { 628 mConnections.put(extensionType, new ExtensionConnection()); 629 } 630 } 631 getProxy(@xtension int extension)632 public ICameraExtensionsProxyService getProxy(@Extension int extension) { 633 return mConnections.get(extension).mProxy; 634 } 635 getConnection(@xtension int extension)636 public ServiceConnection getConnection(@Extension int extension) { 637 return mConnections.get(extension).mConnection; 638 } 639 getConnectionCount(@xtension int extension)640 public int getConnectionCount(@Extension int extension) { 641 return mConnections.get(extension).mConnectionCount; 642 } 643 areAdvancedExtensionsSupported(@xtension int extension)644 public boolean areAdvancedExtensionsSupported(@Extension int extension) { 645 return mConnections.get(extension).mSupportsAdvancedExtensions; 646 } 647 isSessionInitialized()648 public boolean isSessionInitialized() { 649 return mSessionInitialized; 650 } 651 setProxy(@xtension int extension, ICameraExtensionsProxyService proxy)652 public void setProxy(@Extension int extension, ICameraExtensionsProxyService proxy) { 653 mConnections.get(extension).mProxy = proxy; 654 } 655 setConnection(@xtension int extension, ServiceConnection connection)656 public void setConnection(@Extension int extension, ServiceConnection connection) { 657 mConnections.get(extension).mConnection = connection; 658 } 659 incrementConnectionCount(@xtension int extension)660 public void incrementConnectionCount(@Extension int extension) { 661 mConnections.get(extension).mConnectionCount++; 662 } 663 decrementConnectionCount(@xtension int extension)664 public void decrementConnectionCount(@Extension int extension) { 665 mConnections.get(extension).mConnectionCount--; 666 } 667 resetConnectionCount(@xtension int extension)668 public void resetConnectionCount(@Extension int extension) { 669 mConnections.get(extension).mConnectionCount = 0; 670 } 671 setAdvancedExtensionsSupported(@xtension int extension, boolean advancedExtSupported)672 public void setAdvancedExtensionsSupported(@Extension int extension, 673 boolean advancedExtSupported) { 674 mConnections.get(extension).mSupportsAdvancedExtensions = advancedExtSupported; 675 } 676 setSessionInitialized(boolean initialized)677 public void setSessionInitialized(boolean initialized) { 678 mSessionInitialized = initialized; 679 } 680 681 private class ExtensionConnection { 682 public ICameraExtensionsProxyService mProxy = null; 683 public ServiceConnection mConnection = null; 684 public int mConnectionCount = 0; 685 public boolean mSupportsAdvancedExtensions = false; 686 } 687 } 688 } 689 690 /** 691 * @hide 692 */ registerClient(Context ctx, IBinder token, int extension, String cameraId, Map<String, CameraMetadataNative> characteristicsMapNative)693 public static boolean registerClient(Context ctx, IBinder token, int extension, 694 String cameraId, Map<String, CameraMetadataNative> characteristicsMapNative) { 695 return CameraExtensionManagerGlobal.get().registerClient(ctx, token, extension, cameraId, 696 characteristicsMapNative); 697 } 698 699 /** 700 * @hide 701 */ unregisterClient(Context ctx, IBinder token, int extension)702 public static void unregisterClient(Context ctx, IBinder token, int extension) { 703 CameraExtensionManagerGlobal.get().unregisterClient(ctx, token, extension); 704 } 705 706 /** 707 * @hide 708 */ initializeSession(IInitializeSessionCallback cb, int extension)709 public static void initializeSession(IInitializeSessionCallback cb, int extension) 710 throws RemoteException { 711 CameraExtensionManagerGlobal.get().initializeSession(cb, extension); 712 } 713 714 /** 715 * @hide 716 */ releaseSession(int extension)717 public static void releaseSession(int extension) { 718 CameraExtensionManagerGlobal.get().releaseSession(extension); 719 } 720 721 /** 722 * @hide 723 */ areAdvancedExtensionsSupported(int extension)724 public static boolean areAdvancedExtensionsSupported(int extension) { 725 return CameraExtensionManagerGlobal.get().areAdvancedExtensionsSupported(extension); 726 } 727 728 /** 729 * @hide 730 */ isExtensionSupported(String cameraId, int extensionType, Map<String, CameraMetadataNative> characteristicsMap)731 public static boolean isExtensionSupported(String cameraId, int extensionType, 732 Map<String, CameraMetadataNative> characteristicsMap) { 733 if (areAdvancedExtensionsSupported(extensionType)) { 734 try { 735 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extensionType); 736 return extender.isExtensionAvailable(cameraId, characteristicsMap); 737 } catch (RemoteException e) { 738 Log.e(TAG, "Failed to query extension availability! Extension service does not" 739 + " respond!"); 740 return false; 741 } 742 } else { 743 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders; 744 try { 745 extenders = initializeExtension(extensionType); 746 } catch (IllegalArgumentException e) { 747 return false; 748 } 749 750 try { 751 return extenders.first.isExtensionAvailable(cameraId, 752 characteristicsMap.get(cameraId)) 753 && extenders.second.isExtensionAvailable(cameraId, 754 characteristicsMap.get(cameraId)); 755 } catch (RemoteException e) { 756 Log.e(TAG, "Failed to query extension availability! Extension service does not" 757 + " respond!"); 758 return false; 759 } 760 } 761 } 762 763 /** 764 * @hide 765 */ initializeAdvancedExtension(@xtension int extensionType)766 public static IAdvancedExtenderImpl initializeAdvancedExtension(@Extension int extensionType) { 767 IAdvancedExtenderImpl extender; 768 try { 769 extender = CameraExtensionManagerGlobal.get().initializeAdvancedExtension( 770 extensionType); 771 } catch (RemoteException e) { 772 throw new IllegalStateException("Failed to initialize extension: " + extensionType); 773 } 774 775 if (extender == null) { 776 throw new IllegalArgumentException("Unknown extension: " + extensionType); 777 } 778 779 return extender; 780 } 781 782 /** 783 * @hide 784 */ initializeExtension( @xtension int extensionType)785 public static Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> initializeExtension( 786 @Extension int extensionType) { 787 IPreviewExtenderImpl previewExtender; 788 IImageCaptureExtenderImpl imageExtender; 789 try { 790 previewExtender = 791 CameraExtensionManagerGlobal.get().initializePreviewExtension(extensionType); 792 imageExtender = 793 CameraExtensionManagerGlobal.get().initializeImageExtension(extensionType); 794 } catch (RemoteException e) { 795 throw new IllegalStateException("Failed to initialize extension: " + extensionType); 796 } 797 if ((imageExtender == null) || (previewExtender == null)) { 798 throw new IllegalArgumentException("Unknown extension: " + extensionType); 799 } 800 801 return new Pair<>(previewExtender, imageExtender); 802 } 803 isOutputSupportedFor(Class<T> klass)804 private static <T> boolean isOutputSupportedFor(Class<T> klass) { 805 Objects.requireNonNull(klass, "klass must not be null"); 806 807 if ((klass == android.graphics.SurfaceTexture.class) || 808 (klass == android.view.SurfaceView.class)) { 809 return true; 810 } 811 812 return false; 813 } 814 815 /** 816 * Return a list of supported device-specific extensions for a given camera device. 817 * 818 * @return non-modifiable list of available extensions 819 */ getSupportedExtensions()820 public @NonNull List<Integer> getSupportedExtensions() { 821 ArrayList<Integer> ret = new ArrayList<>(); 822 final IBinder token = new Binder(TAG + "#getSupportedExtensions:" + mCameraId); 823 824 IntArray extensionList = new IntArray(EXTENSION_LIST.length); 825 extensionList.addAll(EXTENSION_LIST); 826 827 for (int extensionType : extensionList.toArray()) { 828 try { 829 boolean success = registerClient(mContext, token, extensionType, mCameraId, 830 mCharacteristicsMapNative); 831 if (success && isExtensionSupported(mCameraId, extensionType, 832 mCharacteristicsMapNative)) { 833 ret.add(extensionType); 834 } 835 } finally { 836 unregisterClient(mContext, token, extensionType); 837 } 838 } 839 840 return Collections.unmodifiableList(ret); 841 } 842 843 /** 844 * Gets an extension specific camera characteristics field value. 845 * 846 * <p>An extension can have a reduced set of camera capabilities (such as limited zoom ratio 847 * range, available video stabilization modes, etc). This API enables applications to query for 848 * an extension’s specific camera characteristics. Applications are recommended to prioritize 849 * obtaining camera characteristics using this API when using an extension. A {@code null} 850 * result indicates that the extension specific characteristic is not defined or available. 851 * 852 * @param extension The extension type. 853 * @param key The characteristics field to read. 854 * @return The value of that key, or {@code null} if the field is not set. 855 * 856 * @throws IllegalArgumentException if the key is not valid or extension type is not a supported 857 * device-specific extension. 858 */ 859 @FlaggedApi(Flags.FLAG_CAMERA_EXTENSIONS_CHARACTERISTICS_GET) get(@xtension int extension, @NonNull CameraCharacteristics.Key<T> key)860 public <T> @Nullable T get(@Extension int extension, 861 @NonNull CameraCharacteristics.Key<T> key) { 862 final IBinder token = new Binder(TAG + "#get:" + mCameraId); 863 boolean success = registerClient(mContext, token, extension, mCameraId, 864 mCharacteristicsMapNative); 865 if (!success) { 866 throw new IllegalArgumentException("Unsupported extensions"); 867 } 868 869 try { 870 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 871 throw new IllegalArgumentException("Unsupported extension"); 872 } 873 874 if (areAdvancedExtensionsSupported(extension) && getKeys(extension).contains(key)) { 875 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 876 extender.init(mCameraId, mCharacteristicsMapNative); 877 CameraMetadataNative metadata = 878 extender.getAvailableCharacteristicsKeyValues(mCameraId); 879 if (metadata == null) { 880 return null; 881 } 882 CameraCharacteristics characteristics = new CameraCharacteristics(metadata); 883 return characteristics.get(key); 884 } 885 } catch (RemoteException e) { 886 Log.e(TAG, "Failed to query the extension for the specified key! Extension " 887 + "service does not respond!"); 888 } finally { 889 unregisterClient(mContext, token, extension); 890 } 891 return null; 892 } 893 894 /** 895 * Returns the {@link CameraCharacteristics} keys that have extension-specific values. 896 * 897 * <p>An application can query the value from the key using 898 * {@link #get(int, CameraCharacteristics.Key)} API. 899 * 900 * @param extension The extension type. 901 * @return An unmodifiable set of keys that are extension specific. 902 * 903 * @throws IllegalArgumentException in case the extension type is not a 904 * supported device-specific extension 905 */ 906 @FlaggedApi(Flags.FLAG_CAMERA_EXTENSIONS_CHARACTERISTICS_GET) getKeys(@xtension int extension)907 public @NonNull Set<CameraCharacteristics.Key> getKeys(@Extension int extension) { 908 final IBinder token = 909 new Binder(TAG + "#getKeys:" + mCameraId); 910 boolean success = registerClient(mContext, token, extension, mCameraId, 911 mCharacteristicsMapNative); 912 if (!success) { 913 throw new IllegalArgumentException("Unsupported extensions"); 914 } 915 916 HashSet<CameraCharacteristics.Key> ret = new HashSet<>(); 917 918 try { 919 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 920 throw new IllegalArgumentException("Unsupported extension"); 921 } 922 923 if (areAdvancedExtensionsSupported(extension)) { 924 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 925 extender.init(mCameraId, mCharacteristicsMapNative); 926 CameraMetadataNative metadata = 927 extender.getAvailableCharacteristicsKeyValues(mCameraId); 928 if (metadata == null) { 929 return Collections.emptySet(); 930 } 931 932 int[] keys = metadata.get( 933 CameraCharacteristics.REQUEST_AVAILABLE_CHARACTERISTICS_KEYS); 934 if (keys == null) { 935 throw new AssertionError( 936 "android.request.availableCharacteristicsKeys must be non-null" 937 + " in the characteristics"); 938 } 939 CameraCharacteristics chars = new CameraCharacteristics(metadata); 940 941 Object key = CameraCharacteristics.Key.class; 942 Class<CameraCharacteristics.Key<?>> keyTyped = 943 (Class<CameraCharacteristics.Key<?>>) key; 944 945 ret.addAll(chars.getAvailableKeyList(CameraCharacteristics.class, keyTyped, keys, 946 /*includeSynthetic*/ false)); 947 948 // Add synthetic keys to the available key list if they are part of the supported 949 // synthetic camera characteristic key list 950 for (CameraCharacteristics.Key charKey : 951 SUPPORTED_SYNTHETIC_CAMERA_CHARACTERISTICS) { 952 if (chars.get(charKey) != null) { 953 ret.add(charKey); 954 } 955 } 956 } 957 } catch (RemoteException e) { 958 Log.e(TAG, "Failed to query the extension for all available keys! Extension " 959 + "service does not respond!"); 960 } finally { 961 unregisterClient(mContext, token, extension); 962 } 963 return Collections.unmodifiableSet(ret); 964 } 965 966 /** 967 * Checks for postview support of still capture. 968 * 969 * <p>A postview is a preview version of the still capture that is available before the final 970 * image. For example, it can be used as a temporary placeholder for the requested capture 971 * while the final image is being processed. The supported sizes for a still capture's postview 972 * can be retrieved using 973 * {@link CameraExtensionCharacteristics#getPostviewSupportedSizes(int, Size, int)}.</p> 974 * 975 * <p>Starting with Android {@link android.os.Build.VERSION_CODES#VANILLA_ICE_CREAM}, 976 * the formats of the still capture and postview are not required to be equivalent upon capture 977 * request.</p> 978 * 979 * @param extension the extension type 980 * @return {@code true} in case postview is supported, {@code false} otherwise 981 * 982 * @throws IllegalArgumentException in case the extension type is not a 983 * supported device-specific extension 984 */ isPostviewAvailable(@xtension int extension)985 public boolean isPostviewAvailable(@Extension int extension) { 986 final IBinder token = new Binder(TAG + "#isPostviewAvailable:" + mCameraId); 987 boolean success = registerClient(mContext, token, extension, mCameraId, 988 mCharacteristicsMapNative); 989 if (!success) { 990 throw new IllegalArgumentException("Unsupported extensions"); 991 } 992 993 try { 994 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 995 throw new IllegalArgumentException("Unsupported extension"); 996 } 997 998 if (areAdvancedExtensionsSupported(extension)) { 999 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1000 extender.init(mCameraId, mCharacteristicsMapNative); 1001 return extender.isPostviewAvailable(); 1002 } else { 1003 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1004 initializeExtension(extension); 1005 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1006 return extenders.second.isPostviewAvailable(); 1007 } 1008 } catch (RemoteException e) { 1009 Log.e(TAG, "Failed to query the extension for postview availability! Extension " 1010 + "service does not respond!"); 1011 } finally { 1012 unregisterClient(mContext, token, extension); 1013 } 1014 1015 return false; 1016 } 1017 1018 /** 1019 * Get a list of the postview sizes supported for a still capture, using its 1020 * capture size {@code captureSize}, to use as an output for the postview request. 1021 * 1022 * <p>Available postview sizes will always be either equal to or less than the still 1023 * capture size. When choosing the most applicable postview size for a usecase, it should 1024 * be noted that lower resolution postviews will generally be available more quickly 1025 * than larger resolution postviews. For example, when choosing a size for an optimized 1026 * postview that will be displayed as a placeholder while the final image is processed, 1027 * the resolution closest to the preview size may be most suitable.</p> 1028 * 1029 * <p>Note that device-specific extensions are allowed to support only a subset 1030 * of the camera resolutions advertised by 1031 * {@link StreamConfigurationMap#getOutputSizes}.</p> 1032 * 1033 * @param extension the extension type 1034 * @param captureSize size of the still capture for which the postview is requested 1035 * @param format device-specific extension output format of the postview 1036 * @return non-modifiable list of available sizes or an empty list if the format and 1037 * size is not supported. 1038 * @throws IllegalArgumentException in case of unsupported extension or if postview 1039 * feature is not supported by extension. 1040 */ 1041 @NonNull getPostviewSupportedSizes(@xtension int extension, @NonNull Size captureSize, int format)1042 public List<Size> getPostviewSupportedSizes(@Extension int extension, 1043 @NonNull Size captureSize, int format) { 1044 final IBinder token = new Binder(TAG + "#getPostviewSupportedSizes:" + mCameraId); 1045 boolean success = registerClient(mContext, token, extension, mCameraId, 1046 mCharacteristicsMapNative); 1047 if (!success) { 1048 throw new IllegalArgumentException("Unsupported extensions"); 1049 } 1050 1051 try { 1052 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1053 throw new IllegalArgumentException("Unsupported extension"); 1054 } 1055 1056 android.hardware.camera2.extension.Size sz = 1057 new android.hardware.camera2.extension.Size(); 1058 sz.width = captureSize.getWidth(); 1059 sz.height = captureSize.getHeight(); 1060 1061 StreamConfigurationMap streamMap = mCharacteristicsMap.get(mCameraId).get( 1062 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1063 1064 if (areAdvancedExtensionsSupported(extension)) { 1065 switch(format) { 1066 case ImageFormat.YUV_420_888: 1067 case ImageFormat.JPEG: 1068 case ImageFormat.JPEG_R: 1069 case ImageFormat.DEPTH_JPEG: 1070 case ImageFormat.YCBCR_P010: 1071 break; 1072 default: 1073 throw new IllegalArgumentException("Unsupported format: " + format); 1074 } 1075 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1076 extender.init(mCameraId, mCharacteristicsMapNative); 1077 return getSupportedSizes(extender.getSupportedPostviewResolutions(sz), 1078 format); 1079 } else { 1080 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1081 initializeExtension(extension); 1082 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1083 if ((extenders.second.getCaptureProcessor() == null) || 1084 !isPostviewAvailable(extension)) { 1085 // Extensions that don't implement any capture processor 1086 // and have processing occur in the HAL don't currently support the 1087 // postview feature 1088 throw new IllegalArgumentException("Extension does not support " 1089 + "postview feature"); 1090 } 1091 1092 if (format == ImageFormat.YUV_420_888) { 1093 return getSupportedSizes( 1094 extenders.second.getSupportedPostviewResolutions(sz), format); 1095 } else if (format == ImageFormat.JPEG) { 1096 // The framework will perform the additional encoding pass on the 1097 // processed YUV_420 buffers. 1098 return getSupportedSizes( 1099 extenders.second.getSupportedPostviewResolutions(sz), format); 1100 } else if (format == ImageFormat.JPEG_R || format == ImageFormat.YCBCR_P010 || 1101 (Flags.depthJpegExtensions() && (format == ImageFormat.DEPTH_JPEG))) { 1102 // DepthJpeg/Jpeg_R/UltraHDR + YCBCR_P010 is currently not supported in the 1103 // basic extension case 1104 return new ArrayList<>(); 1105 } else { 1106 throw new IllegalArgumentException("Unsupported format: " + format); 1107 } 1108 } 1109 } catch (RemoteException e) { 1110 Log.e(TAG, "Failed to query the extension postview supported sizes! Extension " 1111 + "service does not respond!"); 1112 return Collections.emptyList(); 1113 } finally { 1114 unregisterClient(mContext, token, extension); 1115 } 1116 } 1117 1118 /** 1119 * Get a list of sizes compatible with {@code klass} to use as an output for the 1120 * repeating request 1121 * {@link CameraExtensionSession#setRepeatingRequest}. 1122 * 1123 * <p>Note that device-specific extensions are allowed to support only a subset 1124 * of the camera output surfaces and resolutions. 1125 * The {@link android.graphics.SurfaceTexture} class is guaranteed at least one size for 1126 * backward compatible cameras whereas other output classes are not guaranteed to be supported. 1127 * </p> 1128 * 1129 * <p>Starting with Android {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE} 1130 * {@link android.view.SurfaceView} classes are also guaranteed to be supported and include 1131 * the same resolutions as {@link android.graphics.SurfaceTexture}. 1132 * Clients must set the desired SurfaceView resolution by calling 1133 * {@link android.view.SurfaceHolder#setFixedSize}.</p> 1134 * 1135 * @param extension the extension type 1136 * @param klass a non-{@code null} {@link Class} object reference 1137 * @return non-modifiable list of available sizes or an empty list if the Surface output is not 1138 * supported 1139 * @throws NullPointerException if {@code klass} was {@code null} 1140 * @throws IllegalArgumentException in case of unsupported extension. 1141 */ 1142 @NonNull getExtensionSupportedSizes(@xtension int extension, @NonNull Class<T> klass)1143 public <T> List<Size> getExtensionSupportedSizes(@Extension int extension, 1144 @NonNull Class<T> klass) { 1145 if (!isOutputSupportedFor(klass)) { 1146 return new ArrayList<>(); 1147 } 1148 // TODO: Revisit this code once the Extension preview processor output format 1149 // ambiguity is resolved in b/169799538. 1150 1151 final IBinder token = new Binder(TAG + "#getExtensionSupportedSizes:" + mCameraId); 1152 boolean success = registerClient(mContext, token, extension, mCameraId, 1153 mCharacteristicsMapNative); 1154 if (!success) { 1155 throw new IllegalArgumentException("Unsupported extensions"); 1156 } 1157 1158 try { 1159 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1160 throw new IllegalArgumentException("Unsupported extension"); 1161 } 1162 1163 StreamConfigurationMap streamMap = mCharacteristicsMap.get(mCameraId).get( 1164 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1165 if (areAdvancedExtensionsSupported(extension)) { 1166 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1167 extender.init(mCameraId, mCharacteristicsMapNative); 1168 return generateSupportedSizes( 1169 extender.getSupportedPreviewOutputResolutions(mCameraId), 1170 ImageFormat.PRIVATE, streamMap); 1171 } else { 1172 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1173 initializeExtension(extension); 1174 extenders.first.init(mCameraId, 1175 mCharacteristicsMapNative.get(mCameraId)); 1176 return generateSupportedSizes(extenders.first.getSupportedResolutions(), 1177 ImageFormat.PRIVATE, streamMap); 1178 } 1179 } catch (RemoteException e) { 1180 Log.e(TAG, "Failed to query the extension supported sizes! Extension service does" 1181 + " not respond!"); 1182 return new ArrayList<>(); 1183 } finally { 1184 unregisterClient(mContext, token, extension); 1185 } 1186 } 1187 1188 /** 1189 * Check whether a given extension is available and return the 1190 * supported output surface resolutions that can be used for high-quality capture 1191 * requests via {@link CameraExtensionSession#capture}. 1192 * 1193 * <p>Note that device-specific extensions are allowed to support only a subset 1194 * of the camera resolutions advertised by 1195 * {@link StreamConfigurationMap#getOutputSizes}.</p> 1196 * 1197 * <p>Device-specific extensions currently support at most three 1198 * multi-frame capture surface formats. ImageFormat.JPEG will be supported by all 1199 * extensions while ImageFormat.YUV_420_888, ImageFormat.JPEG_R, ImageFormat.YCBCR_P010 or 1200 * ImageFormat.DEPTH_JPEG may or may not be supported.</p> 1201 * 1202 * @param extension the extension type 1203 * @param format device-specific extension output format 1204 * @return non-modifiable list of available sizes or an empty list if the format is not 1205 * supported. 1206 * @throws IllegalArgumentException in case of format different from ImageFormat.JPEG, 1207 * ImageFormat.YUV_420_888, ImageFormat.JPEG_R, 1208 * ImageFormat.DEPTH_JPEG, ImageFormat.YCBCR_P010; or 1209 * unsupported extension. 1210 */ 1211 public @NonNull getExtensionSupportedSizes(@xtension int extension, int format)1212 List<Size> getExtensionSupportedSizes(@Extension int extension, int format) { 1213 try { 1214 final IBinder token = new Binder(TAG + "#getExtensionSupportedSizes:" + mCameraId); 1215 boolean success = registerClient(mContext, token, extension, mCameraId, 1216 mCharacteristicsMapNative); 1217 if (!success) { 1218 throw new IllegalArgumentException("Unsupported extensions"); 1219 } 1220 1221 try { 1222 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1223 throw new IllegalArgumentException("Unsupported extension"); 1224 } 1225 1226 StreamConfigurationMap streamMap = mCharacteristicsMap.get(mCameraId).get( 1227 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1228 if (areAdvancedExtensionsSupported(extension)) { 1229 switch(format) { 1230 case ImageFormat.YUV_420_888: 1231 case ImageFormat.JPEG: 1232 case ImageFormat.JPEG_R: 1233 case ImageFormat.DEPTH_JPEG: 1234 case ImageFormat.YCBCR_P010: 1235 break; 1236 default: 1237 throw new IllegalArgumentException("Unsupported format: " + format); 1238 } 1239 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1240 extender.init(mCameraId, mCharacteristicsMapNative); 1241 return generateSupportedSizes(extender.getSupportedCaptureOutputResolutions( 1242 mCameraId), format, streamMap); 1243 } else { 1244 if (format == ImageFormat.YUV_420_888) { 1245 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1246 initializeExtension(extension); 1247 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1248 if (extenders.second.getCaptureProcessor() == null) { 1249 // Extensions that don't implement any capture processor are limited to 1250 // JPEG only! 1251 return new ArrayList<>(); 1252 } 1253 return generateSupportedSizes(extenders.second.getSupportedResolutions(), 1254 format, streamMap); 1255 } else if (format == ImageFormat.JPEG) { 1256 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1257 initializeExtension(extension); 1258 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1259 if (extenders.second.getCaptureProcessor() != null) { 1260 // The framework will perform the additional encoding pass on the 1261 // processed YUV_420 buffers. 1262 return generateJpegSupportedSizes( 1263 extenders.second.getSupportedResolutions(), streamMap); 1264 } else { 1265 return generateSupportedSizes(null, format, streamMap); 1266 } 1267 } else if (format == ImageFormat.JPEG_R || format == ImageFormat.YCBCR_P010 || 1268 (Flags.depthJpegExtensions() && (format == ImageFormat.DEPTH_JPEG))) { 1269 // DepthJpeg/Jpeg_R/UltraHDR + YCBCR_P010 is currently not supported in the 1270 // basic extension case 1271 return new ArrayList<>(); 1272 } else { 1273 throw new IllegalArgumentException("Unsupported format: " + format); 1274 } 1275 } 1276 } finally { 1277 unregisterClient(mContext, token, extension); 1278 } 1279 } catch (RemoteException e) { 1280 Log.e(TAG, "Failed to query the extension supported sizes! Extension service does" 1281 + " not respond!"); 1282 return new ArrayList<>(); 1283 } 1284 } 1285 1286 /** 1287 * Returns the estimated capture latency range in milliseconds for the 1288 * target capture resolution during the calls to {@link CameraExtensionSession#capture}. This 1289 * includes the time spent processing the multi-frame capture request along with any additional 1290 * time for encoding of the processed buffer if necessary. 1291 * 1292 * @param extension the extension type 1293 * @param captureOutputSize size of the capture output surface. If it is not in the supported 1294 * output sizes, maximum capture output size is used for the estimation 1295 * @param format device-specific extension output format 1296 * @return the range of estimated minimal and maximal capture latency in milliseconds 1297 * or null if no capture latency info can be provided 1298 * @throws IllegalArgumentException in case of format different from {@link ImageFormat#JPEG}, 1299 * {@link ImageFormat#YUV_420_888}, {@link ImageFormat#JPEG_R} 1300 * {@link ImageFormat#YCBCR_P010}, 1301 * {@link ImageFormat#DEPTH_JPEG}; 1302 * or unsupported extension. 1303 */ getEstimatedCaptureLatencyRangeMillis(@xtension int extension, @NonNull Size captureOutputSize, @ImageFormat.Format int format)1304 public @Nullable Range<Long> getEstimatedCaptureLatencyRangeMillis(@Extension int extension, 1305 @NonNull Size captureOutputSize, @ImageFormat.Format int format) { 1306 switch (format) { 1307 case ImageFormat.YUV_420_888: 1308 case ImageFormat.JPEG: 1309 case ImageFormat.JPEG_R: 1310 case ImageFormat.DEPTH_JPEG: 1311 case ImageFormat.YCBCR_P010: 1312 //No op 1313 break; 1314 default: 1315 throw new IllegalArgumentException("Unsupported format: " + format); 1316 } 1317 1318 final IBinder token = new Binder(TAG + "#getEstimatedCaptureLatencyRangeMillis:" + mCameraId); 1319 boolean success = registerClient(mContext, token, extension, mCameraId, 1320 mCharacteristicsMapNative); 1321 if (!success) { 1322 throw new IllegalArgumentException("Unsupported extensions"); 1323 } 1324 1325 try { 1326 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1327 throw new IllegalArgumentException("Unsupported extension"); 1328 } 1329 1330 android.hardware.camera2.extension.Size sz = 1331 new android.hardware.camera2.extension.Size(); 1332 sz.width = captureOutputSize.getWidth(); 1333 sz.height = captureOutputSize.getHeight(); 1334 if (areAdvancedExtensionsSupported(extension)) { 1335 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1336 extender.init(mCameraId, mCharacteristicsMapNative); 1337 LatencyRange latencyRange = extender.getEstimatedCaptureLatencyRange(mCameraId, 1338 sz, format); 1339 if (latencyRange != null) { 1340 return new Range(latencyRange.min, latencyRange.max); 1341 } 1342 } else { 1343 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1344 initializeExtension(extension); 1345 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1346 if ((format == ImageFormat.YUV_420_888) && 1347 (extenders.second.getCaptureProcessor() == null) ){ 1348 // Extensions that don't implement any capture processor are limited to 1349 // JPEG only! 1350 return null; 1351 } 1352 if ((format == ImageFormat.JPEG) && 1353 (extenders.second.getCaptureProcessor() != null)) { 1354 // The framework will perform the additional encoding pass on the 1355 // processed YUV_420 buffers. Latency in this case is very device 1356 // specific and cannot be estimated accurately enough. 1357 return null; 1358 } 1359 if (format == ImageFormat.JPEG_R || format == ImageFormat.YCBCR_P010 || 1360 (Flags.depthJpegExtensions() && (format == ImageFormat.DEPTH_JPEG))) { 1361 // DepthJpeg/JpegR/UltraHDR + YCBCR_P010 is not supported for basic extensions 1362 return null; 1363 } 1364 1365 LatencyRange latencyRange = extenders.second.getEstimatedCaptureLatencyRange(sz); 1366 if (latencyRange != null) { 1367 return new Range(latencyRange.min, latencyRange.max); 1368 } 1369 } 1370 } catch (RemoteException e) { 1371 Log.e(TAG, "Failed to query the extension capture latency! Extension service does" 1372 + " not respond!"); 1373 } finally { 1374 unregisterClient(mContext, token, extension); 1375 } 1376 1377 return null; 1378 } 1379 1380 /** 1381 * Retrieve support for capture progress callbacks via 1382 * {@link CameraExtensionSession.ExtensionCaptureCallback#onCaptureProcessProgressed}. 1383 * 1384 * @param extension the extension type 1385 * @return {@code true} in case progress callbacks are supported, {@code false} otherwise 1386 * 1387 * @throws IllegalArgumentException in case of an unsupported extension. 1388 */ isCaptureProcessProgressAvailable(@xtension int extension)1389 public boolean isCaptureProcessProgressAvailable(@Extension int extension) { 1390 final IBinder token = new Binder(TAG + "#isCaptureProcessProgressAvailable:" + mCameraId); 1391 boolean success = registerClient(mContext, token, extension, mCameraId, 1392 mCharacteristicsMapNative); 1393 if (!success) { 1394 throw new IllegalArgumentException("Unsupported extensions"); 1395 } 1396 1397 try { 1398 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1399 throw new IllegalArgumentException("Unsupported extension"); 1400 } 1401 1402 if (areAdvancedExtensionsSupported(extension)) { 1403 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1404 extender.init(mCameraId, mCharacteristicsMapNative); 1405 return extender.isCaptureProcessProgressAvailable(); 1406 } else { 1407 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1408 initializeExtension(extension); 1409 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1410 return extenders.second.isCaptureProcessProgressAvailable(); 1411 } 1412 } catch (RemoteException e) { 1413 Log.e(TAG, "Failed to query the extension progress callbacks! Extension service does" 1414 + " not respond!"); 1415 } finally { 1416 unregisterClient(mContext, token, extension); 1417 } 1418 1419 return false; 1420 } 1421 1422 /** 1423 * Returns the set of keys supported by a {@link CaptureRequest} submitted in a 1424 * {@link CameraExtensionSession} with a given extension type. 1425 * 1426 * <p>The set returned is not modifiable, so any attempts to modify it will throw 1427 * a {@code UnsupportedOperationException}.</p> 1428 * 1429 * <p>Devices launching on Android {@link android.os.Build.VERSION_CODES#VANILLA_ICE_CREAM} 1430 * or newer versions are required to support {@link CaptureRequest#CONTROL_AF_MODE}, 1431 * {@link CaptureRequest#CONTROL_AF_REGIONS}, {@link CaptureRequest#CONTROL_AF_TRIGGER}, 1432 * {@link CaptureRequest#CONTROL_ZOOM_RATIO} for 1433 * {@link CameraExtensionCharacteristics#EXTENSION_NIGHT}.</p> 1434 * 1435 * @param extension the extension type 1436 * 1437 * @return non-modifiable set of capture keys supported by camera extension session initialized 1438 * with the given extension type. 1439 * @throws IllegalArgumentException in case of unsupported extension. 1440 */ 1441 @NonNull getAvailableCaptureRequestKeys(@xtension int extension)1442 public Set<CaptureRequest.Key> getAvailableCaptureRequestKeys(@Extension int extension) { 1443 final IBinder token = new Binder(TAG + "#getAvailableCaptureRequestKeys:" + mCameraId); 1444 boolean success = registerClient(mContext, token, extension, mCameraId, 1445 mCharacteristicsMapNative); 1446 if (!success) { 1447 throw new IllegalArgumentException("Unsupported extensions"); 1448 } 1449 1450 HashSet<CaptureRequest.Key> ret = new HashSet<>(); 1451 1452 try { 1453 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1454 throw new IllegalArgumentException("Unsupported extension"); 1455 } 1456 1457 CameraMetadataNative captureRequestMeta = null; 1458 if (areAdvancedExtensionsSupported(extension)) { 1459 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1460 extender.init(mCameraId, mCharacteristicsMapNative); 1461 captureRequestMeta = extender.getAvailableCaptureRequestKeys(mCameraId); 1462 } else { 1463 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1464 initializeExtension(extension); 1465 extenders.second.onInit(token, mCameraId, 1466 mCharacteristicsMapNative.get(mCameraId)); 1467 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1468 captureRequestMeta = extenders.second.getAvailableCaptureRequestKeys(); 1469 extenders.second.onDeInit(token); 1470 } 1471 1472 if (captureRequestMeta != null) { 1473 int[] requestKeys = captureRequestMeta.get( 1474 CameraCharacteristics.REQUEST_AVAILABLE_REQUEST_KEYS); 1475 if (requestKeys == null) { 1476 throw new AssertionError( 1477 "android.request.availableRequestKeys must be non-null" 1478 + " in the characteristics"); 1479 } 1480 CameraCharacteristics requestChars = new CameraCharacteristics( 1481 captureRequestMeta); 1482 1483 Object crKey = CaptureRequest.Key.class; 1484 Class<CaptureRequest.Key<?>> crKeyTyped = (Class<CaptureRequest.Key<?>>) crKey; 1485 1486 ret.addAll(requestChars.getAvailableKeyList(CaptureRequest.class, crKeyTyped, 1487 requestKeys, /*includeSynthetic*/ true)); 1488 } 1489 1490 // Jpeg quality and orientation must always be supported 1491 if (!ret.contains(CaptureRequest.JPEG_QUALITY)) { 1492 ret.add(CaptureRequest.JPEG_QUALITY); 1493 } 1494 if (!ret.contains(CaptureRequest.JPEG_ORIENTATION)) { 1495 ret.add(CaptureRequest.JPEG_ORIENTATION); 1496 } 1497 } catch (RemoteException e) { 1498 throw new IllegalStateException("Failed to query the available capture request keys!"); 1499 } finally { 1500 unregisterClient(mContext, token, extension); 1501 } 1502 1503 return Collections.unmodifiableSet(ret); 1504 } 1505 1506 /** 1507 * Returns the set of keys supported by a {@link CaptureResult} passed as an argument to 1508 * {@link CameraExtensionSession.ExtensionCaptureCallback#onCaptureResultAvailable}. 1509 * 1510 * <p>The set returned is not modifiable, so any attempts to modify it will throw 1511 * a {@code UnsupportedOperationException}.</p> 1512 * 1513 * <p>In case the set is empty, then the extension is not able to support any capture results 1514 * and the {@link CameraExtensionSession.ExtensionCaptureCallback#onCaptureResultAvailable} 1515 * callback will not be fired.</p> 1516 * 1517 * <p>Devices launching on Android {@link android.os.Build.VERSION_CODES#VANILLA_ICE_CREAM} 1518 * or newer versions are required to support {@link CaptureResult#CONTROL_AF_MODE}, 1519 * {@link CaptureResult#CONTROL_AF_REGIONS}, {@link CaptureResult#CONTROL_AF_TRIGGER}, 1520 * {@link CaptureResult#CONTROL_AF_STATE}, {@link CaptureResult#CONTROL_ZOOM_RATIO} for 1521 * {@link CameraExtensionCharacteristics#EXTENSION_NIGHT}.</p> 1522 * 1523 * @param extension the extension type 1524 * 1525 * @return non-modifiable set of capture result keys supported by camera extension session 1526 * initialized with the given extension type. 1527 * @throws IllegalArgumentException in case of unsupported extension. 1528 */ 1529 @NonNull getAvailableCaptureResultKeys(@xtension int extension)1530 public Set<CaptureResult.Key> getAvailableCaptureResultKeys(@Extension int extension) { 1531 final IBinder token = new Binder(TAG + "#getAvailableCaptureResultKeys:" + mCameraId); 1532 boolean success = registerClient(mContext, token, extension, mCameraId, 1533 mCharacteristicsMapNative); 1534 if (!success) { 1535 throw new IllegalArgumentException("Unsupported extensions"); 1536 } 1537 1538 HashSet<CaptureResult.Key> ret = new HashSet<>(); 1539 try { 1540 if (!isExtensionSupported(mCameraId, extension, mCharacteristicsMapNative)) { 1541 throw new IllegalArgumentException("Unsupported extension"); 1542 } 1543 1544 CameraMetadataNative captureResultMeta = null; 1545 if (areAdvancedExtensionsSupported(extension)) { 1546 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension); 1547 extender.init(mCameraId, mCharacteristicsMapNative); 1548 captureResultMeta = extender.getAvailableCaptureResultKeys(mCameraId); 1549 } else { 1550 Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders = 1551 initializeExtension(extension); 1552 extenders.second.onInit(token, mCameraId, 1553 mCharacteristicsMapNative.get(mCameraId)); 1554 extenders.second.init(mCameraId, mCharacteristicsMapNative.get(mCameraId)); 1555 captureResultMeta = extenders.second.getAvailableCaptureResultKeys(); 1556 extenders.second.onDeInit(token); 1557 } 1558 1559 if (captureResultMeta != null) { 1560 int[] resultKeys = captureResultMeta.get( 1561 CameraCharacteristics.REQUEST_AVAILABLE_RESULT_KEYS); 1562 if (resultKeys == null) { 1563 throw new AssertionError("android.request.availableResultKeys must be non-null " 1564 + "in the characteristics"); 1565 } 1566 CameraCharacteristics resultChars = new CameraCharacteristics(captureResultMeta); 1567 Object crKey = CaptureResult.Key.class; 1568 Class<CaptureResult.Key<?>> crKeyTyped = (Class<CaptureResult.Key<?>>) crKey; 1569 1570 ret.addAll(resultChars.getAvailableKeyList(CaptureResult.class, crKeyTyped, 1571 resultKeys, /*includeSynthetic*/ true)); 1572 1573 // Jpeg quality, orientation and sensor timestamp must always be supported 1574 if (!ret.contains(CaptureResult.JPEG_QUALITY)) { 1575 ret.add(CaptureResult.JPEG_QUALITY); 1576 } 1577 if (!ret.contains(CaptureResult.JPEG_ORIENTATION)) { 1578 ret.add(CaptureResult.JPEG_ORIENTATION); 1579 } 1580 if (!ret.contains(CaptureResult.SENSOR_TIMESTAMP)) { 1581 ret.add(CaptureResult.SENSOR_TIMESTAMP); 1582 } 1583 } 1584 } catch (RemoteException e) { 1585 throw new IllegalStateException("Failed to query the available capture result keys!"); 1586 } finally { 1587 unregisterClient(mContext, token, extension); 1588 } 1589 1590 return Collections.unmodifiableSet(ret); 1591 } 1592 } 1593