1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.annotation.SuppressLint; 23 import android.app.compat.CompatChanges; 24 import android.compat.annotation.ChangeId; 25 import android.compat.annotation.EnabledAfter; 26 import android.graphics.GraphicBuffer; 27 import android.graphics.ImageFormat; 28 import android.graphics.ImageFormat.Format; 29 import android.graphics.Rect; 30 import android.hardware.DataSpace; 31 import android.hardware.DataSpace.NamedDataSpace; 32 import android.hardware.HardwareBuffer; 33 import android.hardware.HardwareBuffer.Usage; 34 import android.hardware.SyncFence; 35 import android.hardware.camera2.MultiResolutionImageReader; 36 import android.os.Build; 37 import android.os.Handler; 38 import android.os.Looper; 39 import android.os.ParcelFileDescriptor; 40 import android.os.Trace; 41 import android.view.Surface; 42 43 import com.android.internal.camera.flags.Flags; 44 45 import dalvik.system.VMRuntime; 46 47 import java.io.IOException; 48 import java.lang.ref.WeakReference; 49 import java.nio.ByteBuffer; 50 import java.nio.ByteOrder; 51 import java.nio.NioUtils; 52 import java.util.List; 53 import java.util.Objects; 54 import java.util.concurrent.CopyOnWriteArrayList; 55 import java.util.concurrent.Executor; 56 import java.util.concurrent.atomic.AtomicBoolean; 57 58 /** 59 * <p>The ImageReader class allows direct application access to image data 60 * rendered into a {@link android.view.Surface}</p> 61 * 62 * <p>Several Android media API classes accept Surface objects as targets to 63 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 64 * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and 65 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 66 * sizes and formats that can be used with each source vary, and should be 67 * checked in the documentation for the specific API.</p> 68 * 69 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 70 * objects can be accessed at the same time, up to the number specified by the 71 * {@code maxImages} constructor parameter. New images sent to an ImageReader 72 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 73 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 74 * eventually stall or drop Images in trying to render to the Surface if the 75 * ImageReader does not obtain and release Images at a rate equal to the 76 * production rate.</p> 77 */ 78 public class ImageReader implements AutoCloseable { 79 80 /** 81 * Returned by nativeImageSetup when acquiring the image was successful. 82 */ 83 private static final int ACQUIRE_SUCCESS = 0; 84 /** 85 * Returned by nativeImageSetup when we couldn't acquire the buffer, 86 * because there were no buffers available to acquire. 87 */ 88 private static final int ACQUIRE_NO_BUFS = 1; 89 /** 90 * Returned by nativeImageSetup when we couldn't acquire the buffer 91 * because the consumer has already acquired {@maxImages} and cannot 92 * acquire more than that. 93 */ 94 private static final int ACQUIRE_MAX_IMAGES = 2; 95 96 /** 97 * <p> 98 * Flag to gate correct exception thrown by {@code #detachImage}. 99 * </p> 100 * <p> 101 * {@code #detachImage} is documented as throwing {@link java.lang.IllegalStateException} in 102 * the event of an error; a native helper method to this threw 103 * {@link java.lang.RuntimeException} if the surface was abandoned while detaching the 104 * {@code Image}. 105 * <p> 106 * This previously undocumented exception behavior continues through Android T. 107 * </p> 108 * <p> 109 * After Android T, the native helper method only throws {@code IllegalStateExceptions} in 110 * accordance with the documentation. 111 * </p> 112 * <p> 113 * {@code #detachImage} will now throw only ISEs if it runs into errors while detaching 114 * the image. Behavior on apps targeting API levels <= T remains unchanged. 115 * </p> 116 */ 117 @ChangeId 118 @EnabledAfter(targetSdkVersion = Build.VERSION_CODES.TIRAMISU) 119 private static final long DETACH_THROWS_ISE_ONLY = 236825255L; 120 121 /** 122 * Cached value of {@link #DETACH_THROWS_ISE_ONLY} flag to prevent repeated calls when 123 * detaching image. 124 */ 125 private final boolean mDetachThrowsIseOnly = 126 CompatChanges.isChangeEnabled(DETACH_THROWS_ISE_ONLY); 127 128 /** 129 * <p> 130 * Create a new reader for images of the desired size and format. 131 * </p> 132 * <p> 133 * The {@code maxImages} parameter determines the maximum number of 134 * {@link Image} objects that can be be acquired from the 135 * {@code ImageReader} simultaneously. Requesting more buffers will use up 136 * more memory, so it is important to use only the minimum number necessary 137 * for the use case. 138 * </p> 139 * <p> 140 * The valid sizes and formats depend on the source of the image data. 141 * </p> 142 * <p> 143 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created 144 * {@link ImageReader} will produce images that are not directly accessible 145 * by the application. The application can still acquire images from this 146 * {@link ImageReader}, and send them to the 147 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via 148 * {@link ImageWriter} interface. However, the {@link Image#getPlanes() 149 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE 150 * PRIVATE} format images. The application can check if an existing reader's 151 * format by calling {@link #getImageFormat()}. 152 * </p> 153 * <p> 154 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader 155 * ImageReaders} are more efficient to use when application access to image 156 * data is not necessary, compared to ImageReaders using other format such 157 * as {@link ImageFormat#YUV_420_888 YUV_420_888}. 158 * </p> 159 * 160 * @param width The default width in pixels of the Images that this reader 161 * will produce. 162 * @param height The default height in pixels of the Images that this reader 163 * will produce. 164 * @param format The format of the Image that this reader will produce. This 165 * must be one of the {@link android.graphics.ImageFormat} or 166 * {@link android.graphics.PixelFormat} constants. Note that not 167 * all formats are supported, like ImageFormat.NV21. 168 * @param maxImages The maximum number of images the user will want to 169 * access simultaneously. This should be as small as possible to 170 * limit memory use. Once maxImages Images are obtained by the 171 * user, one of them has to be released before a new Image will 172 * become available for access through 173 * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 174 * Must be greater than 0. 175 * @see Image 176 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages)177 public static @NonNull ImageReader newInstance( 178 @IntRange(from = 1) int width, 179 @IntRange(from = 1) int height, 180 @Format int format, 181 @IntRange(from = 1) int maxImages) { 182 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 183 // work, and is inscrutable anyway 184 return new ImageReader(width, height, format, maxImages, 185 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, null); 186 } 187 188 /** 189 * <p> 190 * Create a new reader for images of the desired size, format and consumer usage flag. 191 * </p> 192 * <p> 193 * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that 194 * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will 195 * use up more memory, so it is important to use only the minimum number necessary for the use 196 * case. 197 * </p> 198 * <p> 199 * The valid sizes and formats depend on the source of the image data. 200 * </p> 201 * <p> 202 * The format and usage flag combination describes how the buffer will be used by 203 * consumer end-points. For example, if the application intends to send the images to 204 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video 205 * encoding, the format and usage flag combination needs to be 206 * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE_VIDEO_ENCODE}. When an 207 * {@link ImageReader} object is created with a valid size and such format/usage flag 208 * combination, the application can send the {@link Image images} to an {@link ImageWriter} that 209 * is created with the input {@link android.view.Surface} provided by the 210 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. 211 * </p> 212 * <p> 213 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} 214 * will produce images that are not directly accessible by the application. The application can 215 * still acquire images from this {@link ImageReader}, and send them to the 216 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the 217 * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video 218 * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() 219 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format 220 * images. The application can check if an existing reader's format by calling 221 * {@link #getImageFormat()}. 222 * </p> 223 * <p> 224 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more 225 * efficient to use when application access to image data is not necessary, compared to 226 * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. 227 * </p> 228 * <p> 229 * Note that not all format and usage flag combinations are supported by the 230 * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} 231 * (assuming the consumer end-points support the such image consumption, e.g., hardware video 232 * encoding). 233 * <table> 234 * <tr> 235 * <th>Format</th> 236 * <th>Compatible usage flags</th> 237 * </tr> 238 * <tr> 239 * <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by 240 * {@link android.graphics.ImageFormat ImageFormat} or 241 * {@link android.graphics.PixelFormat PixelFormat}</td> 242 * <td>{@link HardwareBuffer#USAGE_CPU_READ_RARELY} or 243 * {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}</td> 244 * </tr> 245 * <tr> 246 * <td>{@link android.graphics.ImageFormat#PRIVATE}</td> 247 * <td>{@link HardwareBuffer#USAGE_VIDEO_ENCODE} or 248 * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}, or combined</td> 249 * </tr> 250 * </table> 251 * Using other combinations may result in {@link IllegalArgumentException}. Additionally, 252 * specifying {@link HardwareBuffer#USAGE_CPU_WRITE_RARELY} or 253 * {@link HardwareBuffer#USAGE_CPU_WRITE_OFTEN} and writing to the ImageReader's buffers 254 * might break assumptions made by some producers, and should be used with caution. 255 * </p> 256 * <p> 257 * If the {@link ImageReader} is used as an output target for a {@link 258 * android.hardware.camera2.CameraDevice}, and if the usage flag contains 259 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE}, the timestamps of the 260 * {@link Image images} produced by the {@link ImageReader} won't be in the same timebase as 261 * {@link android.os.SystemClock#elapsedRealtimeNanos}, even if 262 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is 263 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME}. 264 * Instead, the timestamps will be roughly in the same timebase as in 265 * {@link android.os.SystemClock#uptimeMillis}, so that A/V synchronization could work for 266 * video recording. In this case, the timestamps from the {@link ImageReader} with 267 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE} usage flag may not be directly comparable with 268 * timestamps of other streams or capture result metadata. 269 * </p> 270 * @param width The default width in pixels of the Images that this reader will produce. 271 * @param height The default height in pixels of the Images that this reader will produce. 272 * @param format The format of the Image that this reader will produce. This must be one of the 273 * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} 274 * constants. Note that not all formats are supported, like ImageFormat.NV21. 275 * @param maxImages The maximum number of images the user will want to access simultaneously. 276 * This should be as small as possible to limit memory use. Once maxImages Images are 277 * obtained by the user, one of them has to be released before a new Image will 278 * become available for access through {@link #acquireLatestImage()} or 279 * {@link #acquireNextImage()}. Must be greater than 0. 280 * @param usage The intended usage of the images produced by this ImageReader. See the usages 281 * on {@link HardwareBuffer} for a list of valid usage bits. See also 282 * {@link HardwareBuffer#isSupported(int, int, int, int, long)} for checking 283 * if a combination is supported. If it's not supported this will throw 284 * an {@link IllegalArgumentException}. 285 * @see Image 286 * @see HardwareBuffer 287 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @Usage long usage)288 public static @NonNull ImageReader newInstance( 289 @IntRange(from = 1) int width, 290 @IntRange(from = 1) int height, 291 @Format int format, 292 @IntRange(from = 1) int maxImages, 293 @Usage long usage) { 294 // TODO: Check this - can't do it just yet because format support is different 295 // Unify formats! The only reliable way to validate usage is to just try it and see. 296 297 // if (!HardwareBuffer.isSupported(width, height, format, 1, usage)) { 298 // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) 299 // + " & usage=" + Long.toHexString(usage) + " is not supported"); 300 // } 301 return new ImageReader(width, height, format, maxImages, usage, /*parent*/ null); 302 } 303 304 /** 305 * @hide 306 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @NonNull MultiResolutionImageReader parent)307 public static @NonNull ImageReader newInstance( 308 @IntRange(from = 1) int width, 309 @IntRange(from = 1) int height, 310 @Format int format, 311 @IntRange(from = 1) int maxImages, 312 @NonNull MultiResolutionImageReader parent) { 313 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 314 // work, and is inscrutable anyway 315 return new ImageReader(width, height, format, maxImages, 316 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, parent); 317 } 318 initializeImageReader(int width, int height, int imageFormat, int maxImages, long usage, int hardwareBufferFormat, int dataSpace)319 private void initializeImageReader(int width, int height, int imageFormat, int maxImages, 320 long usage, int hardwareBufferFormat, int dataSpace) { 321 if (width < 1 || height < 1) { 322 throw new IllegalArgumentException( 323 "The image dimensions must be positive"); 324 } 325 326 if (maxImages < 1) { 327 throw new IllegalArgumentException( 328 "Maximum outstanding image count must be at least 1"); 329 } 330 331 if (imageFormat == ImageFormat.NV21) { 332 throw new IllegalArgumentException( 333 "NV21 format is not supported"); 334 } 335 336 nativeInit(new WeakReference<>(this), width, height, maxImages, usage, 337 hardwareBufferFormat, dataSpace); 338 339 mIsReaderValid = true; 340 341 mSurface = nativeGetSurface(); 342 // Estimate the native buffer allocation size and register it so it gets accounted for 343 // during GC. Note that this doesn't include the buffers required by the buffer queue 344 // itself and the buffers requested by the producer. 345 // Only include memory for 1 buffer, since actually accounting for the memory used is 346 // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some 347 // size. 348 mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( 349 width, height, imageFormat, /*buffer count*/ 1); 350 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 351 } 352 ImageReader(int width, int height, int imageFormat, int maxImages, long usage, MultiResolutionImageReader parent)353 private ImageReader(int width, int height, int imageFormat, int maxImages, long usage, 354 MultiResolutionImageReader parent) { 355 mWidth = width; 356 mHeight = height; 357 mFormat = imageFormat; 358 mUsage = usage; 359 mMaxImages = maxImages; 360 mParent = parent; 361 // retrieve hal Format and hal dataspace from imageFormat 362 mHardwareBufferFormat = PublicFormatUtils.getHalFormat(mFormat); 363 mDataSpace = PublicFormatUtils.getHalDataspace(mFormat); 364 mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); 365 366 initializeImageReader(width, height, imageFormat, maxImages, usage, mHardwareBufferFormat, 367 mDataSpace); 368 } 369 ImageReader(int width, int height, int maxImages, long usage, MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace)370 private ImageReader(int width, int height, int maxImages, long usage, 371 MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace) { 372 mWidth = width; 373 mHeight = height; 374 mUsage = usage; 375 mMaxImages = maxImages; 376 mParent = parent; 377 mHardwareBufferFormat = hardwareBufferFormat; 378 mDataSpace = dataSpace; 379 mNumPlanes = ImageUtils.getNumPlanesForHardwareBufferFormat(mHardwareBufferFormat); 380 mFormat = PublicFormatUtils.getPublicFormat(hardwareBufferFormat, dataSpace); 381 382 initializeImageReader(width, height, mFormat, maxImages, usage, hardwareBufferFormat, 383 dataSpace); 384 } 385 386 /** 387 * The default width of {@link Image Images}, in pixels. 388 * 389 * <p>The width may be overridden by the producer sending buffers to this 390 * ImageReader's Surface. If so, the actual width of the images can be 391 * found using {@link Image#getWidth}.</p> 392 * 393 * @return the expected width of an Image 394 */ getWidth()395 public int getWidth() { 396 return mWidth; 397 } 398 399 /** 400 * The default height of {@link Image Images}, in pixels. 401 * 402 * <p>The height may be overridden by the producer sending buffers to this 403 * ImageReader's Surface. If so, the actual height of the images can be 404 * found using {@link Image#getHeight}.</p> 405 * 406 * @return the expected height of an Image 407 */ getHeight()408 public int getHeight() { 409 return mHeight; 410 } 411 412 /** 413 * The default {@link ImageFormat image format} of {@link Image Images}. 414 * 415 * <p>Some color formats may be overridden by the producer sending buffers to 416 * this ImageReader's Surface if the default color format allows. ImageReader 417 * guarantees that all {@link Image Images} acquired from ImageReader 418 * (for example, with {@link #acquireNextImage}) will have a "compatible" 419 * format to what was specified in {@link #newInstance}. 420 * As of now, each format is only compatible to itself. 421 * The actual format of the images can be found using {@link Image#getFormat}.</p> 422 * 423 * <p>Use this function if the ImageReader instance is created by factory method 424 * {@code newInstance} function or by builder pattern {@code ImageReader.Builder} and using 425 * {@link Builder#setImageFormat}.</p> 426 * 427 * @return the expected format of an Image 428 * 429 * @see ImageFormat 430 */ getImageFormat()431 public int getImageFormat() { 432 return mFormat; 433 } 434 435 /** 436 * The default {@link HardwareBuffer} format of {@link Image Images}. 437 * 438 * <p>Use this function if the ImageReader instance is created by builder pattern 439 * {@code ImageReader.Builder} and using {@link Builder#setDefaultHardwareBufferFormat} and 440 * {@link Builder#setDefaultDataSpace}.</p> 441 * 442 * @return the expected {@link HardwareBuffer} format of an Image. 443 */ getHardwareBufferFormat()444 public @HardwareBuffer.Format int getHardwareBufferFormat() { 445 return mHardwareBufferFormat; 446 } 447 448 /** 449 * The default dataspace of {@link Image Images}. 450 * 451 * <p>Use this function if the ImageReader instance is created by builder pattern 452 * {@code ImageReader.Builder} and {@link Builder#setDefaultDataSpace}.</p> 453 * 454 * @return the expected dataspace of an Image. 455 */ 456 @SuppressLint("MethodNameUnits") getDataSpace()457 public @NamedDataSpace int getDataSpace() { 458 return mDataSpace; 459 } 460 461 /** 462 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 463 * with {@link #acquireNextImage}). 464 * 465 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 466 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 467 * </p> 468 * 469 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 470 * acquire function throwing a {@link IllegalStateException}. Furthermore, 471 * while the max number of images have been acquired by the ImageReader user, the producer 472 * enqueueing additional images may stall until at least one image has been released. </p> 473 * 474 * @return Maximum number of images for this ImageReader. 475 * 476 * @see Image#close 477 */ getMaxImages()478 public int getMaxImages() { 479 return mMaxImages; 480 } 481 482 /** 483 * The usage flag of images that can be produced by the ImageReader. 484 * 485 * @return The usage flag of the images for this ImageReader. 486 */ getUsage()487 public @Usage long getUsage() { 488 return mUsage; 489 } 490 491 /** 492 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 493 * {@code ImageReader}.</p> 494 * 495 * <p>Until valid image data is rendered into this {@link Surface}, the 496 * {@link #acquireNextImage} method will return {@code null}. Only one source 497 * can be producing data into this Surface at the same time, although the 498 * same {@link Surface} can be reused with a different API once the first source is 499 * disconnected from the {@link Surface}.</p> 500 * 501 * <p>Please note that holding on to the Surface object returned by this method is not enough 502 * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a 503 * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.</p> 504 * 505 * @return A {@link Surface} to use for a drawing target for various APIs. 506 */ getSurface()507 public Surface getSurface() { 508 return mSurface; 509 } 510 511 /** 512 * <p> 513 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 514 * {@link Image images}. Returns {@code null} if no new image is available. 515 * </p> 516 * <p> 517 * This operation will acquire all the images possible from the ImageReader, 518 * but {@link #close} all images that aren't the latest. This function is 519 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 520 * more suited for real-time processing. 521 * </p> 522 * <p> 523 * Note that {@link #getMaxImages maxImages} should be at least 2 for 524 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 525 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 526 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 527 * with less than two images of margin, that is 528 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 529 * </p> 530 * <p> 531 * This operation will fail by throwing an {@link IllegalStateException} if 532 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 533 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 534 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 535 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 536 * will be thrown until more images are 537 * released with {@link Image#close}. 538 * </p> 539 * 540 * @return latest frame of image data, or {@code null} if no image data is available. 541 * @throws IllegalStateException if too many images are currently acquired 542 */ acquireLatestImage()543 public Image acquireLatestImage() { 544 Image image = acquireNextImage(); 545 if (image == null) { 546 return null; 547 } 548 try { 549 for (;;) { 550 Image next = acquireNextImageNoThrowISE(); 551 if (next == null) { 552 Image result = image; 553 image = null; 554 return result; 555 } 556 image.close(); 557 image = next; 558 } 559 } finally { 560 if (image != null) { 561 image.close(); 562 } 563 if (mParent != null) { 564 mParent.flushOther(this); 565 } 566 } 567 } 568 569 /** 570 * Don't throw IllegalStateException if there are too many images acquired. 571 * 572 * @return Image if acquiring succeeded, or null otherwise. 573 * 574 * @hide 575 */ acquireNextImageNoThrowISE()576 public Image acquireNextImageNoThrowISE() { 577 SurfaceImage si = new SurfaceImage(mFormat); 578 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 579 } 580 581 /** 582 * Attempts to acquire the next image from the underlying native implementation. 583 * 584 * <p> 585 * Note that unexpected failures will throw at the JNI level. 586 * </p> 587 * 588 * @param si A blank SurfaceImage. 589 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 590 * 591 * @see #ACQUIRE_MAX_IMAGES 592 * @see #ACQUIRE_NO_BUFS 593 * @see #ACQUIRE_SUCCESS 594 */ acquireNextSurfaceImage(SurfaceImage si)595 private int acquireNextSurfaceImage(SurfaceImage si) { 596 synchronized (mCloseLock) { 597 // A null image will eventually be returned if ImageReader is already closed. 598 int status = ACQUIRE_NO_BUFS; 599 if (mIsReaderValid) { 600 status = nativeImageSetup(si); 601 } 602 603 switch (status) { 604 case ACQUIRE_SUCCESS: 605 si.mIsImageValid = true; 606 case ACQUIRE_NO_BUFS: 607 case ACQUIRE_MAX_IMAGES: 608 break; 609 default: 610 throw new AssertionError("Unknown nativeImageSetup return code " + status); 611 } 612 613 // Only keep track the successfully acquired image, as the native buffer is only mapped 614 // for such case. 615 if (status == ACQUIRE_SUCCESS) { 616 mAcquiredImages.add(si); 617 } 618 return status; 619 } 620 } 621 622 /** 623 * <p> 624 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 625 * no new image is available. 626 * </p> 627 * 628 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 629 * automatically release older images, and allow slower-running processing routines to catch 630 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 631 * batch/background processing. Incorrectly using this function can cause images to appear 632 * with an ever-increasing delay, followed by a complete stall where no new images seem to 633 * appear. 634 * </p> 635 * 636 * <p> 637 * This operation will fail by throwing an {@link IllegalStateException} if 638 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 639 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 640 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 641 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 642 * {@link IllegalStateException} will be thrown until more images are released with 643 * {@link Image#close}. 644 * </p> 645 * 646 * @return a new frame of image data, or {@code null} if no image data is available. 647 * @throws IllegalStateException if {@code maxImages} images are currently acquired 648 * @see #acquireLatestImage 649 */ acquireNextImage()650 public Image acquireNextImage() { 651 // Initialize with reader format, but can be overwritten by native if the image 652 // format is different from the reader format. 653 SurfaceImage si; 654 si = new SurfaceImage(mFormat); 655 int status = acquireNextSurfaceImage(si); 656 657 switch (status) { 658 case ACQUIRE_SUCCESS: 659 return si; 660 case ACQUIRE_NO_BUFS: 661 return null; 662 case ACQUIRE_MAX_IMAGES: 663 throw new IllegalStateException( 664 String.format( 665 "maxImages (%d) has already been acquired, " + 666 "call #close before acquiring more.", mMaxImages)); 667 default: 668 throw new AssertionError("Unknown nativeImageSetup return code " + status); 669 } 670 } 671 672 /** 673 * <p>Return the frame to the ImageReader for reuse.</p> 674 * 675 * This method should only be called via {@link SurfaceImage#close} which ensures that image 676 * closing is atomic. 677 */ releaseImage(Image i)678 private void releaseImage(Image i) { 679 if (! (i instanceof SurfaceImage) ) { 680 throw new IllegalArgumentException( 681 "This image was not produced by an ImageReader"); 682 } 683 SurfaceImage si = (SurfaceImage) i; 684 if (si.mIsImageValid == false) { 685 return; 686 } 687 688 if (si.getReader() != this || !mAcquiredImages.contains(i)) { 689 throw new IllegalArgumentException( 690 "This image was not produced by this ImageReader"); 691 } 692 693 si.clearSurfacePlanes(); 694 nativeReleaseImage(i); 695 si.mIsImageValid = false; 696 mAcquiredImages.remove(i); 697 } 698 699 /** 700 * Register a listener to be invoked when a new image becomes available 701 * from the ImageReader. 702 * 703 * @param listener 704 * The listener that will be run. 705 * @param handler 706 * The handler on which the listener should be invoked, or null 707 * if the listener should be invoked on the calling thread's looper. 708 * @throws IllegalArgumentException 709 * If no handler specified and the calling thread has no looper. 710 */ setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler)711 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 712 synchronized (mListenerLock) { 713 if (listener != null) { 714 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 715 if (looper == null) { 716 throw new IllegalArgumentException( 717 "handler is null but the current thread is not a looper"); 718 } 719 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 720 mListenerHandler = new ListenerHandler(looper); 721 mListenerExecutor = new HandlerExecutor(mListenerHandler); 722 } 723 } else { 724 mListenerHandler = null; 725 mListenerExecutor = null; 726 } 727 mListener = listener; 728 } 729 } 730 731 /** 732 * Register a listener to be invoked when a new image becomes available 733 * from the ImageReader. 734 * 735 * @param listener 736 * The listener that will be run. 737 * @param executor 738 * The executor which will be used to invoke the listener. 739 * @throws IllegalArgumentException 740 * If no handler specified and the calling thread has no looper. 741 * 742 * @hide 743 */ setOnImageAvailableListenerWithExecutor(@onNull OnImageAvailableListener listener, @NonNull Executor executor)744 public void setOnImageAvailableListenerWithExecutor(@NonNull OnImageAvailableListener listener, 745 @NonNull Executor executor) { 746 if (executor == null) { 747 throw new IllegalArgumentException("executor must not be null"); 748 } 749 750 synchronized (mListenerLock) { 751 mListenerExecutor = executor; 752 mListener = listener; 753 } 754 } 755 756 /** 757 * Callback interface for being notified that a new image is available. 758 * 759 * <p> 760 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 761 * available from ImageReader. 762 * </p> 763 */ 764 public interface OnImageAvailableListener { 765 /** 766 * Callback that is called when a new image is available from ImageReader. 767 * 768 * @param reader the ImageReader the callback is associated with. 769 * @see ImageReader 770 * @see Image 771 */ onImageAvailable(ImageReader reader)772 void onImageAvailable(ImageReader reader); 773 } 774 775 /** 776 * Free up all the resources associated with this ImageReader. 777 * 778 * <p> 779 * After calling this method, this ImageReader can not be used. Calling 780 * any methods on this ImageReader and Images previously provided by 781 * {@link #acquireNextImage} or {@link #acquireLatestImage} 782 * will result in an {@link IllegalStateException}, and attempting to read from 783 * {@link ByteBuffer ByteBuffers} returned by an earlier 784 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 785 * have undefined behavior. 786 * </p> 787 */ 788 @Override close()789 public void close() { 790 setOnImageAvailableListener(null, null); 791 if (mSurface != null) mSurface.release(); 792 793 /** 794 * Close all outstanding acquired images before closing the ImageReader. It is a good 795 * practice to close all the images as soon as it is not used to reduce system instantaneous 796 * memory pressure. CopyOnWrite list will use a copy of current list content. For the images 797 * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For 798 * the image being acquired by other threads, mCloseLock is used to synchronize close and 799 * acquire operations. 800 */ 801 synchronized (mCloseLock) { 802 mIsReaderValid = false; 803 for (Image image : mAcquiredImages) { 804 image.close(); 805 } 806 mAcquiredImages.clear(); 807 808 nativeClose(); 809 810 if (mEstimatedNativeAllocBytes > 0) { 811 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 812 mEstimatedNativeAllocBytes = 0; 813 } 814 } 815 } 816 817 /** 818 * Discard any free buffers owned by this ImageReader. 819 * 820 * <p> 821 * Generally, the ImageReader caches buffers for reuse once they have been 822 * allocated, for best performance. However, sometimes it may be important to 823 * release all the cached, unused buffers to save on memory. 824 * </p> 825 * <p> 826 * Calling this method will discard all free cached buffers. This does not include any buffers 827 * associated with Images acquired from the ImageReader, any filled buffers waiting to be 828 * acquired, and any buffers currently in use by the source rendering buffers into the 829 * ImageReader's Surface. 830 * <p> 831 * The ImageReader continues to be usable after this call, but may need to reallocate buffers 832 * when more buffers are needed for rendering. 833 * </p> 834 */ discardFreeBuffers()835 public void discardFreeBuffers() { 836 synchronized (mCloseLock) { 837 nativeDiscardFreeBuffers(); 838 } 839 } 840 841 @Override finalize()842 protected void finalize() throws Throwable { 843 try { 844 close(); 845 } finally { 846 super.finalize(); 847 } 848 } 849 850 /** 851 * <p> 852 * Remove the ownership of this image from the ImageReader. 853 * </p> 854 * <p> 855 * After this call, the ImageReader no longer owns this image, and the image 856 * ownership can be transferred to another entity like {@link ImageWriter} 857 * via {@link ImageWriter#queueInputImage}. It's up to the new owner to 858 * release the resources held by this image. For example, if the ownership 859 * of this image is transferred to an {@link ImageWriter}, the image will be 860 * freed by the ImageWriter after the image data consumption is done. 861 * </p> 862 * <p> 863 * This method can be used to achieve zero buffer copy for use cases like 864 * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV 865 * reprocessing, where the application can select an output image from 866 * {@link ImageReader} and transfer this image directly to 867 * {@link ImageWriter}, where this image can be consumed by camera directly. 868 * For PRIVATE reprocessing, this is the only way to send input buffers to 869 * the {@link android.hardware.camera2.CameraDevice camera} for 870 * reprocessing. 871 * </p> 872 * <p> 873 * This is a package private method that is only used internally. 874 * </p> 875 * 876 * @param image The image to be detached from this ImageReader. 877 * @throws IllegalStateException If the ImageReader or image have been 878 * closed, or the has been detached, or has not yet been 879 * acquired. 880 * @throws RuntimeException If there is an error detaching {@code Image} from {@code Surface}. 881 * {@code RuntimeException} is only thrown for applications targeting SDK <= 882 * {@link android.os.Build.VERSION_CODES#TIRAMISU}. 883 * For applications targeting SDK > 884 * {@link android.os.Build.VERSION_CODES#TIRAMISU}, 885 * this method only throws {@code IllegalStateException}. 886 * @hide 887 */ detachImage(@ullable Image image)888 public void detachImage(@Nullable Image image) { 889 if (image == null) { 890 throw new IllegalArgumentException("input image must not be null"); 891 } 892 if (!isImageOwnedbyMe(image)) { 893 throw new IllegalArgumentException("Trying to detach an image that is not owned by" 894 + " this ImageReader"); 895 } 896 897 SurfaceImage si = (SurfaceImage) image; 898 si.throwISEIfImageIsInvalid(); 899 900 if (si.isAttachable()) { 901 throw new IllegalStateException("Image was already detached from this ImageReader"); 902 } 903 904 nativeDetachImage(image, mDetachThrowsIseOnly); 905 si.clearSurfacePlanes(); 906 si.mPlanes = null; 907 si.setDetached(true); 908 } 909 isImageOwnedbyMe(Image image)910 private boolean isImageOwnedbyMe(Image image) { 911 if (!(image instanceof SurfaceImage)) { 912 return false; 913 } 914 SurfaceImage si = (SurfaceImage) image; 915 return si.getReader() == this; 916 } 917 918 /** 919 * Called from Native code when an Event happens. 920 * 921 * This may be called from an arbitrary Binder thread, so access to the ImageReader must be 922 * synchronized appropriately. 923 */ postEventFromNative(Object selfRef)924 private static void postEventFromNative(Object selfRef) { 925 @SuppressWarnings("unchecked") 926 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 927 final ImageReader ir = weakSelf.get(); 928 if (ir == null) { 929 return; 930 } 931 Trace.beginSection("android.media.ImageReader#postEventFromNative"); 932 933 final Executor executor; 934 final OnImageAvailableListener listener; 935 synchronized (ir.mListenerLock) { 936 executor = ir.mListenerExecutor; 937 listener = ir.mListener; 938 } 939 final boolean isReaderValid; 940 synchronized (ir.mCloseLock) { 941 isReaderValid = ir.mIsReaderValid; 942 } 943 944 // It's dangerous to fire onImageAvailable() callback when the ImageReader 945 // is being closed, as application could acquire next image in the 946 // onImageAvailable() callback. 947 if (executor != null && listener != null && isReaderValid) { 948 executor.execute(new Runnable() { 949 @Override 950 public void run() { 951 listener.onImageAvailable(ir); 952 } 953 }); 954 } 955 Trace.endSection(); 956 } 957 958 /** 959 * Builder class for {@link ImageReader} objects. 960 */ 961 public static final class Builder { 962 private int mWidth; 963 private int mHeight; 964 private int mMaxImages = 1; 965 private int mImageFormat = ImageFormat.UNKNOWN; 966 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 967 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 968 private long mUsage = HardwareBuffer.USAGE_CPU_READ_OFTEN; 969 private boolean mUseLegacyImageFormat = false; 970 971 /** 972 * Constructs a new builder for {@link ImageReader}. 973 * 974 * @param width The default width in pixels that will be passed to the producer. 975 * May be overridden by the producer. 976 * @param height The default height in pixels that will be passed to the producer. 977 * May be overridden by the producer. 978 * @see Image 979 */ Builder(@ntRangefrom = 1) int width, @IntRange(from = 1) int height)980 public Builder(@IntRange(from = 1) int width, @IntRange(from = 1) int height) { 981 mWidth = width; 982 mHeight = height; 983 } 984 985 /** 986 * Set the maximal number of images. 987 * 988 * @param maxImages The maximum number of images the user will want to 989 * access simultaneously. This should be as small as possible to 990 * limit memory use. Default value is 1. 991 * @return the Builder instance with customized usage value. 992 */ setMaxImages(int maxImages)993 public @NonNull Builder setMaxImages(int maxImages) { 994 mMaxImages = maxImages; 995 return this; 996 } 997 998 /** 999 * Set the consumer usage flag. 1000 * 1001 * @param usage The intended usage of the images consumed by this ImageReader. 1002 * See the usages on {@link HardwareBuffer} for a list of valid usage bits. 1003 * Default value is {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}. 1004 * @return the Builder instance with customized usage value. 1005 * 1006 * @see HardwareBuffer 1007 */ setUsage(@sage long usage)1008 public @NonNull Builder setUsage(@Usage long usage) { 1009 mUsage = usage; 1010 return this; 1011 } 1012 1013 /** 1014 * Set the default image format passed by the producer. May be overridden by the producer. 1015 * 1016 * <p>{@link #setImageFormat} function replaces the combination of 1017 * {@link #setDefaultHardwareBufferFormat} and {@link #setDefaultDataSpace} functions. 1018 * Either this or these two functions must be called to initialize an {@code ImageReader} 1019 * instance.</p> 1020 * 1021 * @param imageFormat The format of the image that this reader will produce. This 1022 * must be one of the {@link android.graphics.ImageFormat} or 1023 * {@link android.graphics.PixelFormat} constants. Note that not 1024 * all formats are supported, like ImageFormat.NV21. The default value is 1025 * {@link ImageFormat#UNKNOWN}. 1026 * @return the builder instance with customized image format value. 1027 * 1028 * @see #setDefaultHardwareBufferFormat 1029 * @see #setDefaultDataSpace 1030 */ setImageFormat(@ormat int imageFormat)1031 public @NonNull Builder setImageFormat(@Format int imageFormat) { 1032 mImageFormat = imageFormat; 1033 mUseLegacyImageFormat = true; 1034 mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 1035 mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1036 return this; 1037 } 1038 1039 /** 1040 * Set the default hardwareBuffer format passed by the producer. 1041 * May be overridden by the producer. 1042 * 1043 * <p>This function works together with {@link #setDefaultDataSpace} for an 1044 * {@link ImageReader} instance. Setting at least one of these two replaces 1045 * {@link #setImageFormat} function.</p> 1046 * 1047 * <p>The format of the Image can be overridden after {@link #setImageFormat} by calling 1048 * this function and then {@link #setDefaultDataSpace} functions. 1049 * <i>Warning:</i> Missing one of callings for initializing or overriding the format may 1050 * involve undefined behaviors.</p> 1051 * 1052 * @param hardwareBufferFormat The HardwareBuffer format of the image that this reader 1053 * will produce. The default value is 1054 * {@link HardwareBuffer#RGBA_8888 HardwareBuffer.RGBA_8888}. 1055 * @return the builder instance with customized hardwareBuffer value. 1056 * 1057 * @see #setDefaultDataSpace 1058 * @see #setImageFormat 1059 */ 1060 @SuppressLint("MissingGetterMatchingBuilder") setDefaultHardwareBufferFormat( @ardwareBuffer.Format int hardwareBufferFormat)1061 public @NonNull Builder setDefaultHardwareBufferFormat( 1062 @HardwareBuffer.Format int hardwareBufferFormat) { 1063 mHardwareBufferFormat = hardwareBufferFormat; 1064 mUseLegacyImageFormat = false; 1065 mImageFormat = ImageFormat.UNKNOWN; 1066 return this; 1067 } 1068 1069 /** 1070 * Set the default dataspace passed by the producer. 1071 * May be overridden by the producer. 1072 * 1073 * <p>This function works together with {@link #setDefaultHardwareBufferFormat} for an 1074 * {@link ImageReader} instance. Setting at least one of these two replaces 1075 * {@link #setImageFormat} function.</p> 1076 * 1077 * @param dataSpace The dataspace of the image that this reader will produce. 1078 * The default value is {@link DataSpace#DATASPACE_UNKNOWN}. 1079 * @return the builder instance with customized dataspace value. 1080 * 1081 * @see #setDefaultHardwareBufferFormat 1082 */ 1083 @SuppressLint("MissingGetterMatchingBuilder") setDefaultDataSpace(@amedDataSpace int dataSpace)1084 public @NonNull Builder setDefaultDataSpace(@NamedDataSpace int dataSpace) { 1085 mDataSpace = dataSpace; 1086 mUseLegacyImageFormat = false; 1087 mImageFormat = ImageFormat.UNKNOWN; 1088 return this; 1089 } 1090 1091 /** 1092 * Builds a new ImageReader object. 1093 * 1094 * @return The new ImageReader object. 1095 */ build()1096 public @NonNull ImageReader build() { 1097 if (mUseLegacyImageFormat) { 1098 return new ImageReader(mWidth, mHeight, mImageFormat, mMaxImages, mUsage, null); 1099 } else { 1100 return new ImageReader(mWidth, mHeight, mMaxImages, mUsage, null, 1101 mHardwareBufferFormat, mDataSpace); 1102 } 1103 } 1104 } 1105 1106 private final int mWidth; 1107 private final int mHeight; 1108 private final int mFormat; 1109 private final long mUsage; 1110 private final int mMaxImages; 1111 private final int mNumPlanes; 1112 private Surface mSurface; 1113 private int mEstimatedNativeAllocBytes; 1114 1115 private final Object mListenerLock = new Object(); 1116 private final Object mCloseLock = new Object(); 1117 private boolean mIsReaderValid = false; 1118 private OnImageAvailableListener mListener; 1119 private Executor mListenerExecutor; 1120 private ListenerHandler mListenerHandler; 1121 // Keep track of the successfully acquired Images. This need to be thread safe as the images 1122 // could be closed by different threads (e.g., application thread and GC thread). 1123 private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>(); 1124 1125 // Applicable if this isn't a standalone ImageReader, but belongs to a 1126 // MultiResolutionImageReader. 1127 private final MultiResolutionImageReader mParent; 1128 1129 private final int mHardwareBufferFormat; 1130 1131 private final @NamedDataSpace int mDataSpace; 1132 1133 /** 1134 * This field is used by native code, do not access or modify. 1135 */ 1136 private long mNativeContext; 1137 1138 /** 1139 * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. 1140 */ 1141 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)1142 public ListenerHandler(Looper looper) { 1143 super(looper, null, true /*async*/); 1144 } 1145 } 1146 1147 /** 1148 * An adapter {@link Executor} that posts all executed tasks onto the 1149 * given {@link Handler}. 1150 **/ 1151 private final class HandlerExecutor implements Executor { 1152 private final Handler mHandler; 1153 HandlerExecutor(@onNull Handler handler)1154 public HandlerExecutor(@NonNull Handler handler) { 1155 mHandler = Objects.requireNonNull(handler); 1156 } 1157 1158 @Override execute(Runnable command)1159 public void execute(Runnable command) { 1160 mHandler.post(command); 1161 } 1162 } 1163 1164 private class SurfaceImage extends android.media.Image { 1165 private final Object mCloseLock = new Object(); 1166 SurfaceImage(int format)1167 public SurfaceImage(int format) { 1168 mFormat = format; 1169 mHardwareBufferFormat = ImageReader.this.mHardwareBufferFormat; 1170 mDataSpace = ImageReader.this.mDataSpace; 1171 } 1172 1173 @Override close()1174 public void close() { 1175 synchronized (this.mCloseLock) { 1176 ImageReader.this.releaseImage(this); 1177 } 1178 } 1179 getReader()1180 public ImageReader getReader() { 1181 return ImageReader.this; 1182 } 1183 1184 @Override getFormat()1185 public int getFormat() { 1186 throwISEIfImageIsInvalid(); 1187 // update mFormat only if ImageReader is initialized by factory pattern. 1188 // if using builder pattern, mFormat has been updated upon initialization. 1189 // no need update here. 1190 int readerFormat = ImageReader.this.getImageFormat(); 1191 // Assume opaque reader always produce opaque images. 1192 mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : 1193 nativeGetFormat(readerFormat); 1194 return mFormat; 1195 } 1196 1197 @Override getWidth()1198 public int getWidth() { 1199 throwISEIfImageIsInvalid(); 1200 int width; 1201 switch(getFormat()) { 1202 case ImageFormat.JPEG: 1203 case ImageFormat.DEPTH_POINT_CLOUD: 1204 case ImageFormat.RAW_PRIVATE: 1205 case ImageFormat.DEPTH_JPEG: 1206 case ImageFormat.HEIC: 1207 case ImageFormat.JPEG_R: 1208 width = ImageReader.this.getWidth(); 1209 break; 1210 default: 1211 width = nativeGetWidth(); 1212 } 1213 if (Flags.cameraHeifGainmap()) { 1214 if (getFormat() == ImageFormat.HEIC_ULTRAHDR){ 1215 width = ImageReader.this.getWidth(); 1216 } 1217 } 1218 return width; 1219 } 1220 1221 @Override getHeight()1222 public int getHeight() { 1223 throwISEIfImageIsInvalid(); 1224 int height; 1225 switch(getFormat()) { 1226 case ImageFormat.JPEG: 1227 case ImageFormat.DEPTH_POINT_CLOUD: 1228 case ImageFormat.RAW_PRIVATE: 1229 case ImageFormat.DEPTH_JPEG: 1230 case ImageFormat.HEIC: 1231 case ImageFormat.JPEG_R: 1232 height = ImageReader.this.getHeight(); 1233 break; 1234 default: 1235 height = nativeGetHeight(); 1236 } 1237 if (Flags.cameraHeifGainmap()) { 1238 if (getFormat() == ImageFormat.HEIC_ULTRAHDR){ 1239 height = ImageReader.this.getHeight(); 1240 } 1241 } 1242 return height; 1243 } 1244 1245 @Override getTimestamp()1246 public long getTimestamp() { 1247 throwISEIfImageIsInvalid(); 1248 return mTimestamp; 1249 } 1250 1251 @Override getTransform()1252 public int getTransform() { 1253 throwISEIfImageIsInvalid(); 1254 return mTransform; 1255 } 1256 1257 @Override getScalingMode()1258 public int getScalingMode() { 1259 throwISEIfImageIsInvalid(); 1260 return mScalingMode; 1261 } 1262 1263 @Override getPlaneCount()1264 public int getPlaneCount() { 1265 throwISEIfImageIsInvalid(); 1266 return ImageReader.this.mNumPlanes; 1267 } 1268 1269 @Override getFence()1270 public SyncFence getFence() throws IOException { 1271 throwISEIfImageIsInvalid(); 1272 // duplicate ParcelFileDescriptor because native still retains the fence ownership. 1273 int fence = nativeGetFenceFd(); 1274 if (fence != -1) { 1275 return SyncFence.create(ParcelFileDescriptor.fromFd(nativeGetFenceFd())); 1276 } else { 1277 return SyncFence.createEmpty(); 1278 } 1279 } 1280 1281 @Override getHardwareBuffer()1282 public HardwareBuffer getHardwareBuffer() { 1283 throwISEIfImageIsInvalid(); 1284 return nativeGetHardwareBuffer(); 1285 } 1286 1287 @Override getDataSpace()1288 public @NamedDataSpace int getDataSpace() { 1289 throwISEIfImageIsInvalid(); 1290 return mDataSpace; 1291 } 1292 1293 @Override setTimestamp(long timestampNs)1294 public void setTimestamp(long timestampNs) { 1295 throwISEIfImageIsInvalid(); 1296 mTimestamp = timestampNs; 1297 } 1298 1299 @Override getPlanes()1300 public Plane[] getPlanes() { 1301 throwISEIfImageIsInvalid(); 1302 1303 if (mPlanes == null) { 1304 mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, 1305 ImageReader.this.mHardwareBufferFormat, ImageReader.this.mUsage); 1306 } 1307 // Shallow copy is fine. 1308 return mPlanes.clone(); 1309 } 1310 1311 @Override finalize()1312 protected final void finalize() throws Throwable { 1313 try { 1314 close(); 1315 } finally { 1316 super.finalize(); 1317 } 1318 } 1319 1320 @Override isAttachable()1321 public boolean isAttachable() { 1322 throwISEIfImageIsInvalid(); 1323 return mIsDetached.get(); 1324 } 1325 1326 @Override getOwner()1327 ImageReader getOwner() { 1328 throwISEIfImageIsInvalid(); 1329 return ImageReader.this; 1330 } 1331 1332 @Override getNativeContext()1333 long getNativeContext() { 1334 throwISEIfImageIsInvalid(); 1335 return mNativeBuffer; 1336 } 1337 setDetached(boolean detached)1338 private void setDetached(boolean detached) { 1339 throwISEIfImageIsInvalid(); 1340 mIsDetached.getAndSet(detached); 1341 } 1342 clearSurfacePlanes()1343 private void clearSurfacePlanes() { 1344 // Image#getPlanes may not be called before the image is closed. 1345 if (mIsImageValid && mPlanes != null) { 1346 for (int i = 0; i < mPlanes.length; i++) { 1347 if (mPlanes[i] != null) { 1348 mPlanes[i].clearBuffer(); 1349 mPlanes[i] = null; 1350 } 1351 } 1352 } 1353 } 1354 1355 private class SurfacePlane extends android.media.Image.Plane { 1356 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 1357 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)1358 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1359 mRowStride = rowStride; 1360 mPixelStride = pixelStride; 1361 mBuffer = buffer; 1362 /** 1363 * Set the byteBuffer order according to host endianness (native 1364 * order), otherwise, the byteBuffer order defaults to 1365 * ByteOrder.BIG_ENDIAN. 1366 */ 1367 mBuffer.order(ByteOrder.nativeOrder()); 1368 } 1369 1370 @Override getBuffer()1371 public ByteBuffer getBuffer() { 1372 throwISEIfImageIsInvalid(); 1373 return mBuffer; 1374 } 1375 1376 @Override getPixelStride()1377 public int getPixelStride() { 1378 SurfaceImage.this.throwISEIfImageIsInvalid(); 1379 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1380 throw new UnsupportedOperationException( 1381 "getPixelStride is not supported for RAW_PRIVATE plane"); 1382 } 1383 return mPixelStride; 1384 } 1385 1386 @Override getRowStride()1387 public int getRowStride() { 1388 SurfaceImage.this.throwISEIfImageIsInvalid(); 1389 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1390 throw new UnsupportedOperationException( 1391 "getRowStride is not supported for RAW_PRIVATE plane"); 1392 } 1393 return mRowStride; 1394 } 1395 clearBuffer()1396 private void clearBuffer() { 1397 // Need null check first, as the getBuffer() may not be called before an image 1398 // is closed. 1399 if (mBuffer == null) { 1400 return; 1401 } 1402 1403 if (mBuffer.isDirect()) { 1404 NioUtils.freeDirectBuffer(mBuffer); 1405 } 1406 mBuffer = null; 1407 } 1408 1409 final private int mPixelStride; 1410 final private int mRowStride; 1411 1412 private ByteBuffer mBuffer; 1413 } 1414 1415 /** 1416 * This field is used to keep track of native object and used by native code only. 1417 * Don't modify. 1418 */ 1419 private long mNativeBuffer; 1420 1421 /** 1422 * These fields are set by native code during nativeImageSetup(). 1423 */ 1424 private long mTimestamp; 1425 private int mTransform; 1426 private int mScalingMode; 1427 1428 private SurfacePlane[] mPlanes; 1429 private int mFormat = ImageFormat.UNKNOWN; 1430 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 1431 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1432 // If this image is detached from the ImageReader. 1433 private AtomicBoolean mIsDetached = new AtomicBoolean(false); 1434 nativeCreatePlanes(int numPlanes, int hardwareBufferFormat, long readerUsage)1435 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, 1436 int hardwareBufferFormat, long readerUsage); nativeGetWidth()1437 private synchronized native int nativeGetWidth(); nativeGetHeight()1438 private synchronized native int nativeGetHeight(); nativeGetFormat(int readerFormat)1439 private synchronized native int nativeGetFormat(int readerFormat); nativeGetFenceFd()1440 private synchronized native int nativeGetFenceFd(); nativeGetHardwareBuffer()1441 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 1442 } 1443 nativeInit(Object weakSelf, int w, int h, int maxImgs, long consumerUsage, int hardwareBufferFormat, int dataSpace)1444 private synchronized native void nativeInit(Object weakSelf, int w, int h, int maxImgs, 1445 long consumerUsage, int hardwareBufferFormat, int dataSpace); nativeClose()1446 private synchronized native void nativeClose(); nativeReleaseImage(Image i)1447 private synchronized native void nativeReleaseImage(Image i); nativeGetSurface()1448 private synchronized native Surface nativeGetSurface(); nativeDetachImage(Image i, boolean throwISEOnly)1449 private synchronized native int nativeDetachImage(Image i, boolean throwISEOnly); nativeDiscardFreeBuffers()1450 private synchronized native void nativeDiscardFreeBuffers(); 1451 1452 /** 1453 * @return A return code {@code ACQUIRE_*} 1454 * 1455 * @see #ACQUIRE_SUCCESS 1456 * @see #ACQUIRE_NO_BUFS 1457 * @see #ACQUIRE_MAX_IMAGES 1458 */ nativeImageSetup(Image i)1459 private synchronized native int nativeImageSetup(Image i); 1460 1461 /** 1462 * @hide 1463 */ 1464 public static class ImagePlane extends android.media.Image.Plane { ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer)1465 private ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1466 mRowStride = rowStride; 1467 mPixelStride = pixelStride; 1468 mBuffer = buffer; 1469 /** 1470 * Set the byteBuffer order according to host endianness (native 1471 * order), otherwise, the byteBuffer order defaults to 1472 * ByteOrder.BIG_ENDIAN. 1473 */ 1474 mBuffer.order(ByteOrder.nativeOrder()); 1475 } 1476 1477 @Override getBuffer()1478 public ByteBuffer getBuffer() { 1479 return mBuffer; 1480 } 1481 1482 @Override getPixelStride()1483 public int getPixelStride() { 1484 return mPixelStride; 1485 } 1486 1487 @Override getRowStride()1488 public int getRowStride() { 1489 return mRowStride; 1490 } 1491 1492 final private int mPixelStride; 1493 final private int mRowStride; 1494 1495 private ByteBuffer mBuffer; 1496 } 1497 1498 /** 1499 * @hide 1500 */ initializeImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, int scalingMode, Rect crop)1501 public static ImagePlane[] initializeImagePlanes(int numPlanes, 1502 GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, 1503 int scalingMode, Rect crop) { 1504 1505 return nativeCreateImagePlanes(numPlanes, buffer, fenceFd, format, crop.left, crop.top, 1506 crop.right, crop.bottom); 1507 } 1508 nativeCreateImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, int cropRight, int cropBottom)1509 private synchronized static native ImagePlane[] nativeCreateImagePlanes(int numPlanes, 1510 GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, 1511 int cropRight, int cropBottom); 1512 1513 /** 1514 * @hide 1515 */ unlockGraphicBuffer(GraphicBuffer buffer)1516 public static void unlockGraphicBuffer(GraphicBuffer buffer) { 1517 nativeUnlockGraphicBuffer(buffer); 1518 } 1519 nativeUnlockGraphicBuffer(GraphicBuffer buffer)1520 private synchronized static native void nativeUnlockGraphicBuffer(GraphicBuffer buffer); 1521 1522 /** 1523 * We use a class initializer to allow the native code to cache some 1524 * field offsets. 1525 */ nativeClassInit()1526 private static native void nativeClassInit(); 1527 static { 1528 System.loadLibrary("media_jni"); nativeClassInit()1529 nativeClassInit(); 1530 } 1531 } 1532