1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.annotation.SuppressLint; 23 import android.app.compat.CompatChanges; 24 import android.compat.annotation.ChangeId; 25 import android.compat.annotation.EnabledAfter; 26 import android.graphics.GraphicBuffer; 27 import android.graphics.ImageFormat; 28 import android.graphics.ImageFormat.Format; 29 import android.graphics.Rect; 30 import android.hardware.DataSpace; 31 import android.hardware.DataSpace.NamedDataSpace; 32 import android.hardware.HardwareBuffer; 33 import android.hardware.HardwareBuffer.Usage; 34 import android.hardware.SyncFence; 35 import android.hardware.camera2.MultiResolutionImageReader; 36 import android.os.Build; 37 import android.os.Handler; 38 import android.os.Looper; 39 import android.os.ParcelFileDescriptor; 40 import android.os.Trace; 41 import android.view.Surface; 42 43 import dalvik.system.VMRuntime; 44 45 import java.io.IOException; 46 import java.lang.ref.WeakReference; 47 import java.nio.ByteBuffer; 48 import java.nio.ByteOrder; 49 import java.nio.NioUtils; 50 import java.util.List; 51 import java.util.Objects; 52 import java.util.concurrent.CopyOnWriteArrayList; 53 import java.util.concurrent.Executor; 54 import java.util.concurrent.atomic.AtomicBoolean; 55 56 /** 57 * <p>The ImageReader class allows direct application access to image data 58 * rendered into a {@link android.view.Surface}</p> 59 * 60 * <p>Several Android media API classes accept Surface objects as targets to 61 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 62 * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and 63 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 64 * sizes and formats that can be used with each source vary, and should be 65 * checked in the documentation for the specific API.</p> 66 * 67 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 68 * objects can be accessed at the same time, up to the number specified by the 69 * {@code maxImages} constructor parameter. New images sent to an ImageReader 70 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 71 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 72 * eventually stall or drop Images in trying to render to the Surface if the 73 * ImageReader does not obtain and release Images at a rate equal to the 74 * production rate.</p> 75 */ 76 public class ImageReader implements AutoCloseable { 77 78 /** 79 * Returned by nativeImageSetup when acquiring the image was successful. 80 */ 81 private static final int ACQUIRE_SUCCESS = 0; 82 /** 83 * Returned by nativeImageSetup when we couldn't acquire the buffer, 84 * because there were no buffers available to acquire. 85 */ 86 private static final int ACQUIRE_NO_BUFS = 1; 87 /** 88 * Returned by nativeImageSetup when we couldn't acquire the buffer 89 * because the consumer has already acquired {@maxImages} and cannot 90 * acquire more than that. 91 */ 92 private static final int ACQUIRE_MAX_IMAGES = 2; 93 94 /** 95 * <p> 96 * Flag to gate correct exception thrown by {@code #detachImage}. 97 * </p> 98 * <p> 99 * {@code #detachImage} is documented as throwing {@link java.lang.IllegalStateException} in 100 * the event of an error; a native helper method to this threw 101 * {@link java.lang.RuntimeException} if the surface was abandoned while detaching the 102 * {@code Image}. 103 * <p> 104 * This previously undocumented exception behavior continues through Android T. 105 * </p> 106 * <p> 107 * After Android T, the native helper method only throws {@code IllegalStateExceptions} in 108 * accordance with the documentation. 109 * </p> 110 * <p> 111 * {@code #detachImage} will now throw only ISEs if it runs into errors while detaching 112 * the image. Behavior on apps targeting API levels <= T remains unchanged. 113 * </p> 114 */ 115 @ChangeId 116 @EnabledAfter(targetSdkVersion = Build.VERSION_CODES.TIRAMISU) 117 private static final long DETACH_THROWS_ISE_ONLY = 236825255L; 118 119 /** 120 * Cached value of {@link #DETACH_THROWS_ISE_ONLY} flag to prevent repeated calls when 121 * detaching image. 122 */ 123 private final boolean mDetachThrowsIseOnly = 124 CompatChanges.isChangeEnabled(DETACH_THROWS_ISE_ONLY); 125 126 /** 127 * <p> 128 * Create a new reader for images of the desired size and format. 129 * </p> 130 * <p> 131 * The {@code maxImages} parameter determines the maximum number of 132 * {@link Image} objects that can be be acquired from the 133 * {@code ImageReader} simultaneously. Requesting more buffers will use up 134 * more memory, so it is important to use only the minimum number necessary 135 * for the use case. 136 * </p> 137 * <p> 138 * The valid sizes and formats depend on the source of the image data. 139 * </p> 140 * <p> 141 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created 142 * {@link ImageReader} will produce images that are not directly accessible 143 * by the application. The application can still acquire images from this 144 * {@link ImageReader}, and send them to the 145 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via 146 * {@link ImageWriter} interface. However, the {@link Image#getPlanes() 147 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE 148 * PRIVATE} format images. The application can check if an existing reader's 149 * format by calling {@link #getImageFormat()}. 150 * </p> 151 * <p> 152 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader 153 * ImageReaders} are more efficient to use when application access to image 154 * data is not necessary, compared to ImageReaders using other format such 155 * as {@link ImageFormat#YUV_420_888 YUV_420_888}. 156 * </p> 157 * 158 * @param width The default width in pixels of the Images that this reader 159 * will produce. 160 * @param height The default height in pixels of the Images that this reader 161 * will produce. 162 * @param format The format of the Image that this reader will produce. This 163 * must be one of the {@link android.graphics.ImageFormat} or 164 * {@link android.graphics.PixelFormat} constants. Note that not 165 * all formats are supported, like ImageFormat.NV21. 166 * @param maxImages The maximum number of images the user will want to 167 * access simultaneously. This should be as small as possible to 168 * limit memory use. Once maxImages Images are obtained by the 169 * user, one of them has to be released before a new Image will 170 * become available for access through 171 * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 172 * Must be greater than 0. 173 * @see Image 174 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages)175 public static @NonNull ImageReader newInstance( 176 @IntRange(from = 1) int width, 177 @IntRange(from = 1) int height, 178 @Format int format, 179 @IntRange(from = 1) int maxImages) { 180 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 181 // work, and is inscrutable anyway 182 return new ImageReader(width, height, format, maxImages, 183 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, null); 184 } 185 186 /** 187 * <p> 188 * Create a new reader for images of the desired size, format and consumer usage flag. 189 * </p> 190 * <p> 191 * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that 192 * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will 193 * use up more memory, so it is important to use only the minimum number necessary for the use 194 * case. 195 * </p> 196 * <p> 197 * The valid sizes and formats depend on the source of the image data. 198 * </p> 199 * <p> 200 * The format and usage flag combination describes how the buffer will be used by 201 * consumer end-points. For example, if the application intends to send the images to 202 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video 203 * encoding, the format and usage flag combination needs to be 204 * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE_VIDEO_ENCODE}. When an 205 * {@link ImageReader} object is created with a valid size and such format/usage flag 206 * combination, the application can send the {@link Image images} to an {@link ImageWriter} that 207 * is created with the input {@link android.view.Surface} provided by the 208 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. 209 * </p> 210 * <p> 211 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} 212 * will produce images that are not directly accessible by the application. The application can 213 * still acquire images from this {@link ImageReader}, and send them to the 214 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the 215 * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video 216 * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() 217 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format 218 * images. The application can check if an existing reader's format by calling 219 * {@link #getImageFormat()}. 220 * </p> 221 * <p> 222 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more 223 * efficient to use when application access to image data is not necessary, compared to 224 * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. 225 * </p> 226 * <p> 227 * Note that not all format and usage flag combinations are supported by the 228 * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} 229 * (assuming the consumer end-points support the such image consumption, e.g., hardware video 230 * encoding). 231 * <table> 232 * <tr> 233 * <th>Format</th> 234 * <th>Compatible usage flags</th> 235 * </tr> 236 * <tr> 237 * <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by 238 * {@link android.graphics.ImageFormat ImageFormat} or 239 * {@link android.graphics.PixelFormat PixelFormat}</td> 240 * <td>{@link HardwareBuffer#USAGE_CPU_READ_RARELY} or 241 * {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}</td> 242 * </tr> 243 * <tr> 244 * <td>{@link android.graphics.ImageFormat#PRIVATE}</td> 245 * <td>{@link HardwareBuffer#USAGE_VIDEO_ENCODE} or 246 * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}, or combined</td> 247 * </tr> 248 * </table> 249 * Using other combinations may result in {@link IllegalArgumentException}. Additionally, 250 * specifying {@link HardwareBuffer#USAGE_CPU_WRITE_RARELY} or 251 * {@link HardwareBuffer#USAGE_CPU_WRITE_OFTEN} and writing to the ImageReader's buffers 252 * might break assumptions made by some producers, and should be used with caution. 253 * </p> 254 * <p> 255 * If the {@link ImageReader} is used as an output target for a {@link 256 * android.hardware.camera2.CameraDevice}, and if the usage flag contains 257 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE}, the timestamps of the 258 * {@link Image images} produced by the {@link ImageReader} won't be in the same timebase as 259 * {@link android.os.SystemClock#elapsedRealtimeNanos}, even if 260 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is 261 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME}. 262 * Instead, the timestamps will be roughly in the same timebase as in 263 * {@link android.os.SystemClock#uptimeMillis}, so that A/V synchronization could work for 264 * video recording. In this case, the timestamps from the {@link ImageReader} with 265 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE} usage flag may not be directly comparable with 266 * timestamps of other streams or capture result metadata. 267 * </p> 268 * @param width The default width in pixels of the Images that this reader will produce. 269 * @param height The default height in pixels of the Images that this reader will produce. 270 * @param format The format of the Image that this reader will produce. This must be one of the 271 * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} 272 * constants. Note that not all formats are supported, like ImageFormat.NV21. 273 * @param maxImages The maximum number of images the user will want to access simultaneously. 274 * This should be as small as possible to limit memory use. Once maxImages Images are 275 * obtained by the user, one of them has to be released before a new Image will 276 * become available for access through {@link #acquireLatestImage()} or 277 * {@link #acquireNextImage()}. Must be greater than 0. 278 * @param usage The intended usage of the images produced by this ImageReader. See the usages 279 * on {@link HardwareBuffer} for a list of valid usage bits. See also 280 * {@link HardwareBuffer#isSupported(int, int, int, int, long)} for checking 281 * if a combination is supported. If it's not supported this will throw 282 * an {@link IllegalArgumentException}. 283 * @see Image 284 * @see HardwareBuffer 285 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @Usage long usage)286 public static @NonNull ImageReader newInstance( 287 @IntRange(from = 1) int width, 288 @IntRange(from = 1) int height, 289 @Format int format, 290 @IntRange(from = 1) int maxImages, 291 @Usage long usage) { 292 // TODO: Check this - can't do it just yet because format support is different 293 // Unify formats! The only reliable way to validate usage is to just try it and see. 294 295 // if (!HardwareBuffer.isSupported(width, height, format, 1, usage)) { 296 // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) 297 // + " & usage=" + Long.toHexString(usage) + " is not supported"); 298 // } 299 return new ImageReader(width, height, format, maxImages, usage, /*parent*/ null); 300 } 301 302 /** 303 * @hide 304 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @NonNull MultiResolutionImageReader parent)305 public static @NonNull ImageReader newInstance( 306 @IntRange(from = 1) int width, 307 @IntRange(from = 1) int height, 308 @Format int format, 309 @IntRange(from = 1) int maxImages, 310 @NonNull MultiResolutionImageReader parent) { 311 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 312 // work, and is inscrutable anyway 313 return new ImageReader(width, height, format, maxImages, 314 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, parent); 315 } 316 initializeImageReader(int width, int height, int imageFormat, int maxImages, long usage, int hardwareBufferFormat, int dataSpace)317 private void initializeImageReader(int width, int height, int imageFormat, int maxImages, 318 long usage, int hardwareBufferFormat, int dataSpace) { 319 if (width < 1 || height < 1) { 320 throw new IllegalArgumentException( 321 "The image dimensions must be positive"); 322 } 323 324 if (maxImages < 1) { 325 throw new IllegalArgumentException( 326 "Maximum outstanding image count must be at least 1"); 327 } 328 329 if (imageFormat == ImageFormat.NV21) { 330 throw new IllegalArgumentException( 331 "NV21 format is not supported"); 332 } 333 334 nativeInit(new WeakReference<>(this), width, height, maxImages, usage, 335 hardwareBufferFormat, dataSpace); 336 337 mIsReaderValid = true; 338 339 mSurface = nativeGetSurface(); 340 // Estimate the native buffer allocation size and register it so it gets accounted for 341 // during GC. Note that this doesn't include the buffers required by the buffer queue 342 // itself and the buffers requested by the producer. 343 // Only include memory for 1 buffer, since actually accounting for the memory used is 344 // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some 345 // size. 346 mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( 347 width, height, imageFormat, /*buffer count*/ 1); 348 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 349 } 350 ImageReader(int width, int height, int imageFormat, int maxImages, long usage, MultiResolutionImageReader parent)351 private ImageReader(int width, int height, int imageFormat, int maxImages, long usage, 352 MultiResolutionImageReader parent) { 353 mWidth = width; 354 mHeight = height; 355 mFormat = imageFormat; 356 mUsage = usage; 357 mMaxImages = maxImages; 358 mParent = parent; 359 // retrieve hal Format and hal dataspace from imageFormat 360 mHardwareBufferFormat = PublicFormatUtils.getHalFormat(mFormat); 361 mDataSpace = PublicFormatUtils.getHalDataspace(mFormat); 362 mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); 363 364 initializeImageReader(width, height, imageFormat, maxImages, usage, mHardwareBufferFormat, 365 mDataSpace); 366 } 367 ImageReader(int width, int height, int maxImages, long usage, MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace)368 private ImageReader(int width, int height, int maxImages, long usage, 369 MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace) { 370 mWidth = width; 371 mHeight = height; 372 mUsage = usage; 373 mMaxImages = maxImages; 374 mParent = parent; 375 mHardwareBufferFormat = hardwareBufferFormat; 376 mDataSpace = dataSpace; 377 mNumPlanes = ImageUtils.getNumPlanesForHardwareBufferFormat(mHardwareBufferFormat); 378 mFormat = PublicFormatUtils.getPublicFormat(hardwareBufferFormat, dataSpace); 379 380 initializeImageReader(width, height, mFormat, maxImages, usage, hardwareBufferFormat, 381 dataSpace); 382 } 383 384 /** 385 * The default width of {@link Image Images}, in pixels. 386 * 387 * <p>The width may be overridden by the producer sending buffers to this 388 * ImageReader's Surface. If so, the actual width of the images can be 389 * found using {@link Image#getWidth}.</p> 390 * 391 * @return the expected width of an Image 392 */ getWidth()393 public int getWidth() { 394 return mWidth; 395 } 396 397 /** 398 * The default height of {@link Image Images}, in pixels. 399 * 400 * <p>The height may be overridden by the producer sending buffers to this 401 * ImageReader's Surface. If so, the actual height of the images can be 402 * found using {@link Image#getHeight}.</p> 403 * 404 * @return the expected height of an Image 405 */ getHeight()406 public int getHeight() { 407 return mHeight; 408 } 409 410 /** 411 * The default {@link ImageFormat image format} of {@link Image Images}. 412 * 413 * <p>Some color formats may be overridden by the producer sending buffers to 414 * this ImageReader's Surface if the default color format allows. ImageReader 415 * guarantees that all {@link Image Images} acquired from ImageReader 416 * (for example, with {@link #acquireNextImage}) will have a "compatible" 417 * format to what was specified in {@link #newInstance}. 418 * As of now, each format is only compatible to itself. 419 * The actual format of the images can be found using {@link Image#getFormat}.</p> 420 * 421 * <p>Use this function if the ImageReader instance is created by factory method 422 * {@code newInstance} function or by builder pattern {@code ImageReader.Builder} and using 423 * {@link Builder#setImageFormat}.</p> 424 * 425 * @return the expected format of an Image 426 * 427 * @see ImageFormat 428 */ getImageFormat()429 public int getImageFormat() { 430 return mFormat; 431 } 432 433 /** 434 * The default {@link HardwareBuffer} format of {@link Image Images}. 435 * 436 * <p>Use this function if the ImageReader instance is created by builder pattern 437 * {@code ImageReader.Builder} and using {@link Builder#setDefaultHardwareBufferFormat} and 438 * {@link Builder#setDefaultDataSpace}.</p> 439 * 440 * @return the expected {@link HardwareBuffer} format of an Image. 441 */ getHardwareBufferFormat()442 public @HardwareBuffer.Format int getHardwareBufferFormat() { 443 return mHardwareBufferFormat; 444 } 445 446 /** 447 * The default dataspace of {@link Image Images}. 448 * 449 * <p>Use this function if the ImageReader instance is created by builder pattern 450 * {@code ImageReader.Builder} and {@link Builder#setDefaultDataSpace}.</p> 451 * 452 * @return the expected dataspace of an Image. 453 */ 454 @SuppressLint("MethodNameUnits") getDataSpace()455 public @NamedDataSpace int getDataSpace() { 456 return mDataSpace; 457 } 458 459 /** 460 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 461 * with {@link #acquireNextImage}). 462 * 463 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 464 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 465 * </p> 466 * 467 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 468 * acquire function throwing a {@link IllegalStateException}. Furthermore, 469 * while the max number of images have been acquired by the ImageReader user, the producer 470 * enqueueing additional images may stall until at least one image has been released. </p> 471 * 472 * @return Maximum number of images for this ImageReader. 473 * 474 * @see Image#close 475 */ getMaxImages()476 public int getMaxImages() { 477 return mMaxImages; 478 } 479 480 /** 481 * The usage flag of images that can be produced by the ImageReader. 482 * 483 * @return The usage flag of the images for this ImageReader. 484 */ getUsage()485 public @Usage long getUsage() { 486 return mUsage; 487 } 488 489 /** 490 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 491 * {@code ImageReader}.</p> 492 * 493 * <p>Until valid image data is rendered into this {@link Surface}, the 494 * {@link #acquireNextImage} method will return {@code null}. Only one source 495 * can be producing data into this Surface at the same time, although the 496 * same {@link Surface} can be reused with a different API once the first source is 497 * disconnected from the {@link Surface}.</p> 498 * 499 * <p>Please note that holding on to the Surface object returned by this method is not enough 500 * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a 501 * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.</p> 502 * 503 * @return A {@link Surface} to use for a drawing target for various APIs. 504 */ getSurface()505 public Surface getSurface() { 506 return mSurface; 507 } 508 509 /** 510 * <p> 511 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 512 * {@link Image images}. Returns {@code null} if no new image is available. 513 * </p> 514 * <p> 515 * This operation will acquire all the images possible from the ImageReader, 516 * but {@link #close} all images that aren't the latest. This function is 517 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 518 * more suited for real-time processing. 519 * </p> 520 * <p> 521 * Note that {@link #getMaxImages maxImages} should be at least 2 for 522 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 523 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 524 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 525 * with less than two images of margin, that is 526 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 527 * </p> 528 * <p> 529 * This operation will fail by throwing an {@link IllegalStateException} if 530 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 531 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 532 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 533 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 534 * will be thrown until more images are 535 * released with {@link Image#close}. 536 * </p> 537 * 538 * @return latest frame of image data, or {@code null} if no image data is available. 539 * @throws IllegalStateException if too many images are currently acquired 540 */ acquireLatestImage()541 public Image acquireLatestImage() { 542 Image image = acquireNextImage(); 543 if (image == null) { 544 return null; 545 } 546 try { 547 for (;;) { 548 Image next = acquireNextImageNoThrowISE(); 549 if (next == null) { 550 Image result = image; 551 image = null; 552 return result; 553 } 554 image.close(); 555 image = next; 556 } 557 } finally { 558 if (image != null) { 559 image.close(); 560 } 561 if (mParent != null) { 562 mParent.flushOther(this); 563 } 564 } 565 } 566 567 /** 568 * Don't throw IllegalStateException if there are too many images acquired. 569 * 570 * @return Image if acquiring succeeded, or null otherwise. 571 * 572 * @hide 573 */ acquireNextImageNoThrowISE()574 public Image acquireNextImageNoThrowISE() { 575 SurfaceImage si = new SurfaceImage(mFormat); 576 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 577 } 578 579 /** 580 * Attempts to acquire the next image from the underlying native implementation. 581 * 582 * <p> 583 * Note that unexpected failures will throw at the JNI level. 584 * </p> 585 * 586 * @param si A blank SurfaceImage. 587 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 588 * 589 * @see #ACQUIRE_MAX_IMAGES 590 * @see #ACQUIRE_NO_BUFS 591 * @see #ACQUIRE_SUCCESS 592 */ acquireNextSurfaceImage(SurfaceImage si)593 private int acquireNextSurfaceImage(SurfaceImage si) { 594 synchronized (mCloseLock) { 595 // A null image will eventually be returned if ImageReader is already closed. 596 int status = ACQUIRE_NO_BUFS; 597 if (mIsReaderValid) { 598 status = nativeImageSetup(si); 599 } 600 601 switch (status) { 602 case ACQUIRE_SUCCESS: 603 si.mIsImageValid = true; 604 case ACQUIRE_NO_BUFS: 605 case ACQUIRE_MAX_IMAGES: 606 break; 607 default: 608 throw new AssertionError("Unknown nativeImageSetup return code " + status); 609 } 610 611 // Only keep track the successfully acquired image, as the native buffer is only mapped 612 // for such case. 613 if (status == ACQUIRE_SUCCESS) { 614 mAcquiredImages.add(si); 615 } 616 return status; 617 } 618 } 619 620 /** 621 * <p> 622 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 623 * no new image is available. 624 * </p> 625 * 626 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 627 * automatically release older images, and allow slower-running processing routines to catch 628 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 629 * batch/background processing. Incorrectly using this function can cause images to appear 630 * with an ever-increasing delay, followed by a complete stall where no new images seem to 631 * appear. 632 * </p> 633 * 634 * <p> 635 * This operation will fail by throwing an {@link IllegalStateException} if 636 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 637 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 638 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 639 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 640 * {@link IllegalStateException} will be thrown until more images are released with 641 * {@link Image#close}. 642 * </p> 643 * 644 * @return a new frame of image data, or {@code null} if no image data is available. 645 * @throws IllegalStateException if {@code maxImages} images are currently acquired 646 * @see #acquireLatestImage 647 */ acquireNextImage()648 public Image acquireNextImage() { 649 // Initialize with reader format, but can be overwritten by native if the image 650 // format is different from the reader format. 651 SurfaceImage si; 652 si = new SurfaceImage(mFormat); 653 int status = acquireNextSurfaceImage(si); 654 655 switch (status) { 656 case ACQUIRE_SUCCESS: 657 return si; 658 case ACQUIRE_NO_BUFS: 659 return null; 660 case ACQUIRE_MAX_IMAGES: 661 throw new IllegalStateException( 662 String.format( 663 "maxImages (%d) has already been acquired, " + 664 "call #close before acquiring more.", mMaxImages)); 665 default: 666 throw new AssertionError("Unknown nativeImageSetup return code " + status); 667 } 668 } 669 670 /** 671 * <p>Return the frame to the ImageReader for reuse.</p> 672 * 673 * This method should only be called via {@link SurfaceImage#close} which ensures that image 674 * closing is atomic. 675 */ releaseImage(Image i)676 private void releaseImage(Image i) { 677 if (! (i instanceof SurfaceImage) ) { 678 throw new IllegalArgumentException( 679 "This image was not produced by an ImageReader"); 680 } 681 SurfaceImage si = (SurfaceImage) i; 682 if (si.mIsImageValid == false) { 683 return; 684 } 685 686 if (si.getReader() != this || !mAcquiredImages.contains(i)) { 687 throw new IllegalArgumentException( 688 "This image was not produced by this ImageReader"); 689 } 690 691 si.clearSurfacePlanes(); 692 nativeReleaseImage(i); 693 si.mIsImageValid = false; 694 mAcquiredImages.remove(i); 695 } 696 697 /** 698 * Register a listener to be invoked when a new image becomes available 699 * from the ImageReader. 700 * 701 * @param listener 702 * The listener that will be run. 703 * @param handler 704 * The handler on which the listener should be invoked, or null 705 * if the listener should be invoked on the calling thread's looper. 706 * @throws IllegalArgumentException 707 * If no handler specified and the calling thread has no looper. 708 */ setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler)709 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 710 synchronized (mListenerLock) { 711 if (listener != null) { 712 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 713 if (looper == null) { 714 throw new IllegalArgumentException( 715 "handler is null but the current thread is not a looper"); 716 } 717 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 718 mListenerHandler = new ListenerHandler(looper); 719 mListenerExecutor = new HandlerExecutor(mListenerHandler); 720 } 721 } else { 722 mListenerHandler = null; 723 mListenerExecutor = null; 724 } 725 mListener = listener; 726 } 727 } 728 729 /** 730 * Register a listener to be invoked when a new image becomes available 731 * from the ImageReader. 732 * 733 * @param listener 734 * The listener that will be run. 735 * @param executor 736 * The executor which will be used to invoke the listener. 737 * @throws IllegalArgumentException 738 * If no handler specified and the calling thread has no looper. 739 * 740 * @hide 741 */ setOnImageAvailableListenerWithExecutor(@onNull OnImageAvailableListener listener, @NonNull Executor executor)742 public void setOnImageAvailableListenerWithExecutor(@NonNull OnImageAvailableListener listener, 743 @NonNull Executor executor) { 744 if (executor == null) { 745 throw new IllegalArgumentException("executor must not be null"); 746 } 747 748 synchronized (mListenerLock) { 749 mListenerExecutor = executor; 750 mListener = listener; 751 } 752 } 753 754 /** 755 * Callback interface for being notified that a new image is available. 756 * 757 * <p> 758 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 759 * available from ImageReader. 760 * </p> 761 */ 762 public interface OnImageAvailableListener { 763 /** 764 * Callback that is called when a new image is available from ImageReader. 765 * 766 * @param reader the ImageReader the callback is associated with. 767 * @see ImageReader 768 * @see Image 769 */ onImageAvailable(ImageReader reader)770 void onImageAvailable(ImageReader reader); 771 } 772 773 /** 774 * Free up all the resources associated with this ImageReader. 775 * 776 * <p> 777 * After calling this method, this ImageReader can not be used. Calling 778 * any methods on this ImageReader and Images previously provided by 779 * {@link #acquireNextImage} or {@link #acquireLatestImage} 780 * will result in an {@link IllegalStateException}, and attempting to read from 781 * {@link ByteBuffer ByteBuffers} returned by an earlier 782 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 783 * have undefined behavior. 784 * </p> 785 */ 786 @Override close()787 public void close() { 788 setOnImageAvailableListener(null, null); 789 if (mSurface != null) mSurface.release(); 790 791 /** 792 * Close all outstanding acquired images before closing the ImageReader. It is a good 793 * practice to close all the images as soon as it is not used to reduce system instantaneous 794 * memory pressure. CopyOnWrite list will use a copy of current list content. For the images 795 * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For 796 * the image being acquired by other threads, mCloseLock is used to synchronize close and 797 * acquire operations. 798 */ 799 synchronized (mCloseLock) { 800 mIsReaderValid = false; 801 for (Image image : mAcquiredImages) { 802 image.close(); 803 } 804 mAcquiredImages.clear(); 805 806 nativeClose(); 807 808 if (mEstimatedNativeAllocBytes > 0) { 809 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 810 mEstimatedNativeAllocBytes = 0; 811 } 812 } 813 } 814 815 /** 816 * Discard any free buffers owned by this ImageReader. 817 * 818 * <p> 819 * Generally, the ImageReader caches buffers for reuse once they have been 820 * allocated, for best performance. However, sometimes it may be important to 821 * release all the cached, unused buffers to save on memory. 822 * </p> 823 * <p> 824 * Calling this method will discard all free cached buffers. This does not include any buffers 825 * associated with Images acquired from the ImageReader, any filled buffers waiting to be 826 * acquired, and any buffers currently in use by the source rendering buffers into the 827 * ImageReader's Surface. 828 * <p> 829 * The ImageReader continues to be usable after this call, but may need to reallocate buffers 830 * when more buffers are needed for rendering. 831 * </p> 832 */ discardFreeBuffers()833 public void discardFreeBuffers() { 834 synchronized (mCloseLock) { 835 nativeDiscardFreeBuffers(); 836 } 837 } 838 839 @Override finalize()840 protected void finalize() throws Throwable { 841 try { 842 close(); 843 } finally { 844 super.finalize(); 845 } 846 } 847 848 /** 849 * <p> 850 * Remove the ownership of this image from the ImageReader. 851 * </p> 852 * <p> 853 * After this call, the ImageReader no longer owns this image, and the image 854 * ownership can be transferred to another entity like {@link ImageWriter} 855 * via {@link ImageWriter#queueInputImage}. It's up to the new owner to 856 * release the resources held by this image. For example, if the ownership 857 * of this image is transferred to an {@link ImageWriter}, the image will be 858 * freed by the ImageWriter after the image data consumption is done. 859 * </p> 860 * <p> 861 * This method can be used to achieve zero buffer copy for use cases like 862 * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV 863 * reprocessing, where the application can select an output image from 864 * {@link ImageReader} and transfer this image directly to 865 * {@link ImageWriter}, where this image can be consumed by camera directly. 866 * For PRIVATE reprocessing, this is the only way to send input buffers to 867 * the {@link android.hardware.camera2.CameraDevice camera} for 868 * reprocessing. 869 * </p> 870 * <p> 871 * This is a package private method that is only used internally. 872 * </p> 873 * 874 * @param image The image to be detached from this ImageReader. 875 * @throws IllegalStateException If the ImageReader or image have been 876 * closed, or the has been detached, or has not yet been 877 * acquired. 878 * @throws RuntimeException If there is an error detaching {@code Image} from {@code Surface}. 879 * {@code RuntimeException} is only thrown for applications targeting SDK <= 880 * {@link android.os.Build.VERSION_CODES#TIRAMISU}. 881 * For applications targeting SDK > 882 * {@link android.os.Build.VERSION_CODES#TIRAMISU}, 883 * this method only throws {@code IllegalStateException}. 884 * @hide 885 */ detachImage(@ullable Image image)886 public void detachImage(@Nullable Image image) { 887 if (image == null) { 888 throw new IllegalArgumentException("input image must not be null"); 889 } 890 if (!isImageOwnedbyMe(image)) { 891 throw new IllegalArgumentException("Trying to detach an image that is not owned by" 892 + " this ImageReader"); 893 } 894 895 SurfaceImage si = (SurfaceImage) image; 896 si.throwISEIfImageIsInvalid(); 897 898 if (si.isAttachable()) { 899 throw new IllegalStateException("Image was already detached from this ImageReader"); 900 } 901 902 nativeDetachImage(image, mDetachThrowsIseOnly); 903 si.clearSurfacePlanes(); 904 si.mPlanes = null; 905 si.setDetached(true); 906 } 907 isImageOwnedbyMe(Image image)908 private boolean isImageOwnedbyMe(Image image) { 909 if (!(image instanceof SurfaceImage)) { 910 return false; 911 } 912 SurfaceImage si = (SurfaceImage) image; 913 return si.getReader() == this; 914 } 915 916 /** 917 * Called from Native code when an Event happens. 918 * 919 * This may be called from an arbitrary Binder thread, so access to the ImageReader must be 920 * synchronized appropriately. 921 */ postEventFromNative(Object selfRef)922 private static void postEventFromNative(Object selfRef) { 923 @SuppressWarnings("unchecked") 924 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 925 final ImageReader ir = weakSelf.get(); 926 if (ir == null) { 927 return; 928 } 929 Trace.beginSection("android.media.ImageReader#postEventFromNative"); 930 931 final Executor executor; 932 final OnImageAvailableListener listener; 933 synchronized (ir.mListenerLock) { 934 executor = ir.mListenerExecutor; 935 listener = ir.mListener; 936 } 937 final boolean isReaderValid; 938 synchronized (ir.mCloseLock) { 939 isReaderValid = ir.mIsReaderValid; 940 } 941 942 // It's dangerous to fire onImageAvailable() callback when the ImageReader 943 // is being closed, as application could acquire next image in the 944 // onImageAvailable() callback. 945 if (executor != null && listener != null && isReaderValid) { 946 executor.execute(new Runnable() { 947 @Override 948 public void run() { 949 listener.onImageAvailable(ir); 950 } 951 }); 952 } 953 Trace.endSection(); 954 } 955 956 /** 957 * Builder class for {@link ImageReader} objects. 958 */ 959 public static final class Builder { 960 private int mWidth; 961 private int mHeight; 962 private int mMaxImages = 1; 963 private int mImageFormat = ImageFormat.UNKNOWN; 964 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 965 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 966 private long mUsage = HardwareBuffer.USAGE_CPU_READ_OFTEN; 967 private boolean mUseLegacyImageFormat = false; 968 969 /** 970 * Constructs a new builder for {@link ImageReader}. 971 * 972 * @param width The default width in pixels that will be passed to the producer. 973 * May be overridden by the producer. 974 * @param height The default height in pixels that will be passed to the producer. 975 * May be overridden by the producer. 976 * @see Image 977 */ Builder(@ntRangefrom = 1) int width, @IntRange(from = 1) int height)978 public Builder(@IntRange(from = 1) int width, @IntRange(from = 1) int height) { 979 mWidth = width; 980 mHeight = height; 981 } 982 983 /** 984 * Set the maximal number of images. 985 * 986 * @param maxImages The maximum number of images the user will want to 987 * access simultaneously. This should be as small as possible to 988 * limit memory use. Default value is 1. 989 * @return the Builder instance with customized usage value. 990 */ setMaxImages(int maxImages)991 public @NonNull Builder setMaxImages(int maxImages) { 992 mMaxImages = maxImages; 993 return this; 994 } 995 996 /** 997 * Set the consumer usage flag. 998 * 999 * @param usage The intended usage of the images consumed by this ImageReader. 1000 * See the usages on {@link HardwareBuffer} for a list of valid usage bits. 1001 * Default value is {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}. 1002 * @return the Builder instance with customized usage value. 1003 * 1004 * @see HardwareBuffer 1005 */ setUsage(@sage long usage)1006 public @NonNull Builder setUsage(@Usage long usage) { 1007 mUsage = usage; 1008 return this; 1009 } 1010 1011 /** 1012 * Set the default image format passed by the producer. May be overridden by the producer. 1013 * 1014 * <p>{@link #setImageFormat} function replaces the combination of 1015 * {@link #setDefaultHardwareBufferFormat} and {@link #setDefaultDataSpace} functions. 1016 * Either this or these two functions must be called to initialize an {@code ImageReader} 1017 * instance.</p> 1018 * 1019 * @param imageFormat The format of the image that this reader will produce. This 1020 * must be one of the {@link android.graphics.ImageFormat} or 1021 * {@link android.graphics.PixelFormat} constants. Note that not 1022 * all formats are supported, like ImageFormat.NV21. The default value is 1023 * {@link ImageFormat#UNKNOWN}. 1024 * @return the builder instance with customized image format value. 1025 * 1026 * @see #setDefaultHardwareBufferFormat 1027 * @see #setDefaultDataSpace 1028 */ setImageFormat(@ormat int imageFormat)1029 public @NonNull Builder setImageFormat(@Format int imageFormat) { 1030 mImageFormat = imageFormat; 1031 mUseLegacyImageFormat = true; 1032 mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 1033 mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1034 return this; 1035 } 1036 1037 /** 1038 * Set the default hardwareBuffer format passed by the producer. 1039 * May be overridden by the producer. 1040 * 1041 * <p>This function works together with {@link #setDefaultDataSpace} for an 1042 * {@link ImageReader} instance. Setting at least one of these two replaces 1043 * {@link #setImageFormat} function.</p> 1044 * 1045 * <p>The format of the Image can be overridden after {@link #setImageFormat} by calling 1046 * this function and then {@link #setDefaultDataSpace} functions. 1047 * <i>Warning:</i> Missing one of callings for initializing or overriding the format may 1048 * involve undefined behaviors.</p> 1049 * 1050 * @param hardwareBufferFormat The HardwareBuffer format of the image that this reader 1051 * will produce. The default value is 1052 * {@link HardwareBuffer#RGBA_8888 HardwareBuffer.RGBA_8888}. 1053 * @return the builder instance with customized hardwareBuffer value. 1054 * 1055 * @see #setDefaultDataSpace 1056 * @see #setImageFormat 1057 */ 1058 @SuppressLint("MissingGetterMatchingBuilder") setDefaultHardwareBufferFormat( @ardwareBuffer.Format int hardwareBufferFormat)1059 public @NonNull Builder setDefaultHardwareBufferFormat( 1060 @HardwareBuffer.Format int hardwareBufferFormat) { 1061 mHardwareBufferFormat = hardwareBufferFormat; 1062 mUseLegacyImageFormat = false; 1063 mImageFormat = ImageFormat.UNKNOWN; 1064 return this; 1065 } 1066 1067 /** 1068 * Set the default dataspace passed by the producer. 1069 * May be overridden by the producer. 1070 * 1071 * <p>This function works together with {@link #setDefaultHardwareBufferFormat} for an 1072 * {@link ImageReader} instance. Setting at least one of these two replaces 1073 * {@link #setImageFormat} function.</p> 1074 * 1075 * @param dataSpace The dataspace of the image that this reader will produce. 1076 * The default value is {@link DataSpace#DATASPACE_UNKNOWN}. 1077 * @return the builder instance with customized dataspace value. 1078 * 1079 * @see #setDefaultHardwareBufferFormat 1080 */ 1081 @SuppressLint("MissingGetterMatchingBuilder") setDefaultDataSpace(@amedDataSpace int dataSpace)1082 public @NonNull Builder setDefaultDataSpace(@NamedDataSpace int dataSpace) { 1083 mDataSpace = dataSpace; 1084 mUseLegacyImageFormat = false; 1085 mImageFormat = ImageFormat.UNKNOWN; 1086 return this; 1087 } 1088 1089 /** 1090 * Builds a new ImageReader object. 1091 * 1092 * @return The new ImageReader object. 1093 */ build()1094 public @NonNull ImageReader build() { 1095 if (mUseLegacyImageFormat) { 1096 return new ImageReader(mWidth, mHeight, mImageFormat, mMaxImages, mUsage, null); 1097 } else { 1098 return new ImageReader(mWidth, mHeight, mMaxImages, mUsage, null, 1099 mHardwareBufferFormat, mDataSpace); 1100 } 1101 } 1102 } 1103 1104 private final int mWidth; 1105 private final int mHeight; 1106 private final int mFormat; 1107 private final long mUsage; 1108 private final int mMaxImages; 1109 private final int mNumPlanes; 1110 private Surface mSurface; 1111 private int mEstimatedNativeAllocBytes; 1112 1113 private final Object mListenerLock = new Object(); 1114 private final Object mCloseLock = new Object(); 1115 private boolean mIsReaderValid = false; 1116 private OnImageAvailableListener mListener; 1117 private Executor mListenerExecutor; 1118 private ListenerHandler mListenerHandler; 1119 // Keep track of the successfully acquired Images. This need to be thread safe as the images 1120 // could be closed by different threads (e.g., application thread and GC thread). 1121 private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>(); 1122 1123 // Applicable if this isn't a standalone ImageReader, but belongs to a 1124 // MultiResolutionImageReader. 1125 private final MultiResolutionImageReader mParent; 1126 1127 private final int mHardwareBufferFormat; 1128 1129 private final @NamedDataSpace int mDataSpace; 1130 1131 /** 1132 * This field is used by native code, do not access or modify. 1133 */ 1134 private long mNativeContext; 1135 1136 /** 1137 * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. 1138 */ 1139 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)1140 public ListenerHandler(Looper looper) { 1141 super(looper, null, true /*async*/); 1142 } 1143 } 1144 1145 /** 1146 * An adapter {@link Executor} that posts all executed tasks onto the 1147 * given {@link Handler}. 1148 **/ 1149 private final class HandlerExecutor implements Executor { 1150 private final Handler mHandler; 1151 HandlerExecutor(@onNull Handler handler)1152 public HandlerExecutor(@NonNull Handler handler) { 1153 mHandler = Objects.requireNonNull(handler); 1154 } 1155 1156 @Override execute(Runnable command)1157 public void execute(Runnable command) { 1158 mHandler.post(command); 1159 } 1160 } 1161 1162 private class SurfaceImage extends android.media.Image { 1163 private final Object mCloseLock = new Object(); 1164 SurfaceImage(int format)1165 public SurfaceImage(int format) { 1166 mFormat = format; 1167 mHardwareBufferFormat = ImageReader.this.mHardwareBufferFormat; 1168 mDataSpace = ImageReader.this.mDataSpace; 1169 } 1170 1171 @Override close()1172 public void close() { 1173 synchronized (this.mCloseLock) { 1174 ImageReader.this.releaseImage(this); 1175 } 1176 } 1177 getReader()1178 public ImageReader getReader() { 1179 return ImageReader.this; 1180 } 1181 1182 @Override getFormat()1183 public int getFormat() { 1184 throwISEIfImageIsInvalid(); 1185 // update mFormat only if ImageReader is initialized by factory pattern. 1186 // if using builder pattern, mFormat has been updated upon initialization. 1187 // no need update here. 1188 int readerFormat = ImageReader.this.getImageFormat(); 1189 // Assume opaque reader always produce opaque images. 1190 mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : 1191 nativeGetFormat(readerFormat); 1192 return mFormat; 1193 } 1194 1195 @Override getWidth()1196 public int getWidth() { 1197 throwISEIfImageIsInvalid(); 1198 int width; 1199 switch(getFormat()) { 1200 case ImageFormat.JPEG: 1201 case ImageFormat.DEPTH_POINT_CLOUD: 1202 case ImageFormat.RAW_PRIVATE: 1203 case ImageFormat.DEPTH_JPEG: 1204 case ImageFormat.HEIC: 1205 case ImageFormat.JPEG_R: 1206 width = ImageReader.this.getWidth(); 1207 break; 1208 default: 1209 width = nativeGetWidth(); 1210 } 1211 return width; 1212 } 1213 1214 @Override getHeight()1215 public int getHeight() { 1216 throwISEIfImageIsInvalid(); 1217 int height; 1218 switch(getFormat()) { 1219 case ImageFormat.JPEG: 1220 case ImageFormat.DEPTH_POINT_CLOUD: 1221 case ImageFormat.RAW_PRIVATE: 1222 case ImageFormat.DEPTH_JPEG: 1223 case ImageFormat.HEIC: 1224 case ImageFormat.JPEG_R: 1225 height = ImageReader.this.getHeight(); 1226 break; 1227 default: 1228 height = nativeGetHeight(); 1229 } 1230 return height; 1231 } 1232 1233 @Override getTimestamp()1234 public long getTimestamp() { 1235 throwISEIfImageIsInvalid(); 1236 return mTimestamp; 1237 } 1238 1239 @Override getTransform()1240 public int getTransform() { 1241 throwISEIfImageIsInvalid(); 1242 return mTransform; 1243 } 1244 1245 @Override getScalingMode()1246 public int getScalingMode() { 1247 throwISEIfImageIsInvalid(); 1248 return mScalingMode; 1249 } 1250 1251 @Override getPlaneCount()1252 public int getPlaneCount() { 1253 throwISEIfImageIsInvalid(); 1254 return ImageReader.this.mNumPlanes; 1255 } 1256 1257 @Override getFence()1258 public SyncFence getFence() throws IOException { 1259 throwISEIfImageIsInvalid(); 1260 // duplicate ParcelFileDescriptor because native still retains the fence ownership. 1261 int fence = nativeGetFenceFd(); 1262 if (fence != -1) { 1263 return SyncFence.create(ParcelFileDescriptor.fromFd(nativeGetFenceFd())); 1264 } else { 1265 return SyncFence.createEmpty(); 1266 } 1267 } 1268 1269 @Override getHardwareBuffer()1270 public HardwareBuffer getHardwareBuffer() { 1271 throwISEIfImageIsInvalid(); 1272 return nativeGetHardwareBuffer(); 1273 } 1274 1275 @Override getDataSpace()1276 public @NamedDataSpace int getDataSpace() { 1277 throwISEIfImageIsInvalid(); 1278 return mDataSpace; 1279 } 1280 1281 @Override setTimestamp(long timestampNs)1282 public void setTimestamp(long timestampNs) { 1283 throwISEIfImageIsInvalid(); 1284 mTimestamp = timestampNs; 1285 } 1286 1287 @Override getPlanes()1288 public Plane[] getPlanes() { 1289 throwISEIfImageIsInvalid(); 1290 1291 if (mPlanes == null) { 1292 mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, 1293 ImageReader.this.mHardwareBufferFormat, ImageReader.this.mUsage); 1294 } 1295 // Shallow copy is fine. 1296 return mPlanes.clone(); 1297 } 1298 1299 @Override finalize()1300 protected final void finalize() throws Throwable { 1301 try { 1302 close(); 1303 } finally { 1304 super.finalize(); 1305 } 1306 } 1307 1308 @Override isAttachable()1309 public boolean isAttachable() { 1310 throwISEIfImageIsInvalid(); 1311 return mIsDetached.get(); 1312 } 1313 1314 @Override getOwner()1315 ImageReader getOwner() { 1316 throwISEIfImageIsInvalid(); 1317 return ImageReader.this; 1318 } 1319 1320 @Override getNativeContext()1321 long getNativeContext() { 1322 throwISEIfImageIsInvalid(); 1323 return mNativeBuffer; 1324 } 1325 setDetached(boolean detached)1326 private void setDetached(boolean detached) { 1327 throwISEIfImageIsInvalid(); 1328 mIsDetached.getAndSet(detached); 1329 } 1330 clearSurfacePlanes()1331 private void clearSurfacePlanes() { 1332 // Image#getPlanes may not be called before the image is closed. 1333 if (mIsImageValid && mPlanes != null) { 1334 for (int i = 0; i < mPlanes.length; i++) { 1335 if (mPlanes[i] != null) { 1336 mPlanes[i].clearBuffer(); 1337 mPlanes[i] = null; 1338 } 1339 } 1340 } 1341 } 1342 1343 private class SurfacePlane extends android.media.Image.Plane { 1344 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 1345 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)1346 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1347 mRowStride = rowStride; 1348 mPixelStride = pixelStride; 1349 mBuffer = buffer; 1350 /** 1351 * Set the byteBuffer order according to host endianness (native 1352 * order), otherwise, the byteBuffer order defaults to 1353 * ByteOrder.BIG_ENDIAN. 1354 */ 1355 mBuffer.order(ByteOrder.nativeOrder()); 1356 } 1357 1358 @Override getBuffer()1359 public ByteBuffer getBuffer() { 1360 throwISEIfImageIsInvalid(); 1361 return mBuffer; 1362 } 1363 1364 @Override getPixelStride()1365 public int getPixelStride() { 1366 SurfaceImage.this.throwISEIfImageIsInvalid(); 1367 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1368 throw new UnsupportedOperationException( 1369 "getPixelStride is not supported for RAW_PRIVATE plane"); 1370 } 1371 return mPixelStride; 1372 } 1373 1374 @Override getRowStride()1375 public int getRowStride() { 1376 SurfaceImage.this.throwISEIfImageIsInvalid(); 1377 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1378 throw new UnsupportedOperationException( 1379 "getRowStride is not supported for RAW_PRIVATE plane"); 1380 } 1381 return mRowStride; 1382 } 1383 clearBuffer()1384 private void clearBuffer() { 1385 // Need null check first, as the getBuffer() may not be called before an image 1386 // is closed. 1387 if (mBuffer == null) { 1388 return; 1389 } 1390 1391 if (mBuffer.isDirect()) { 1392 NioUtils.freeDirectBuffer(mBuffer); 1393 } 1394 mBuffer = null; 1395 } 1396 1397 final private int mPixelStride; 1398 final private int mRowStride; 1399 1400 private ByteBuffer mBuffer; 1401 } 1402 1403 /** 1404 * This field is used to keep track of native object and used by native code only. 1405 * Don't modify. 1406 */ 1407 private long mNativeBuffer; 1408 1409 /** 1410 * These fields are set by native code during nativeImageSetup(). 1411 */ 1412 private long mTimestamp; 1413 private int mTransform; 1414 private int mScalingMode; 1415 1416 private SurfacePlane[] mPlanes; 1417 private int mFormat = ImageFormat.UNKNOWN; 1418 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 1419 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1420 // If this image is detached from the ImageReader. 1421 private AtomicBoolean mIsDetached = new AtomicBoolean(false); 1422 nativeCreatePlanes(int numPlanes, int hardwareBufferFormat, long readerUsage)1423 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, 1424 int hardwareBufferFormat, long readerUsage); nativeGetWidth()1425 private synchronized native int nativeGetWidth(); nativeGetHeight()1426 private synchronized native int nativeGetHeight(); nativeGetFormat(int readerFormat)1427 private synchronized native int nativeGetFormat(int readerFormat); nativeGetFenceFd()1428 private synchronized native int nativeGetFenceFd(); nativeGetHardwareBuffer()1429 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 1430 } 1431 nativeInit(Object weakSelf, int w, int h, int maxImgs, long consumerUsage, int hardwareBufferFormat, int dataSpace)1432 private synchronized native void nativeInit(Object weakSelf, int w, int h, int maxImgs, 1433 long consumerUsage, int hardwareBufferFormat, int dataSpace); nativeClose()1434 private synchronized native void nativeClose(); nativeReleaseImage(Image i)1435 private synchronized native void nativeReleaseImage(Image i); nativeGetSurface()1436 private synchronized native Surface nativeGetSurface(); nativeDetachImage(Image i, boolean throwISEOnly)1437 private synchronized native int nativeDetachImage(Image i, boolean throwISEOnly); nativeDiscardFreeBuffers()1438 private synchronized native void nativeDiscardFreeBuffers(); 1439 1440 /** 1441 * @return A return code {@code ACQUIRE_*} 1442 * 1443 * @see #ACQUIRE_SUCCESS 1444 * @see #ACQUIRE_NO_BUFS 1445 * @see #ACQUIRE_MAX_IMAGES 1446 */ nativeImageSetup(Image i)1447 private synchronized native int nativeImageSetup(Image i); 1448 1449 /** 1450 * @hide 1451 */ 1452 public static class ImagePlane extends android.media.Image.Plane { ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer)1453 private ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1454 mRowStride = rowStride; 1455 mPixelStride = pixelStride; 1456 mBuffer = buffer; 1457 /** 1458 * Set the byteBuffer order according to host endianness (native 1459 * order), otherwise, the byteBuffer order defaults to 1460 * ByteOrder.BIG_ENDIAN. 1461 */ 1462 mBuffer.order(ByteOrder.nativeOrder()); 1463 } 1464 1465 @Override getBuffer()1466 public ByteBuffer getBuffer() { 1467 return mBuffer; 1468 } 1469 1470 @Override getPixelStride()1471 public int getPixelStride() { 1472 return mPixelStride; 1473 } 1474 1475 @Override getRowStride()1476 public int getRowStride() { 1477 return mRowStride; 1478 } 1479 1480 final private int mPixelStride; 1481 final private int mRowStride; 1482 1483 private ByteBuffer mBuffer; 1484 } 1485 1486 /** 1487 * @hide 1488 */ initializeImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, int scalingMode, Rect crop)1489 public static ImagePlane[] initializeImagePlanes(int numPlanes, 1490 GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, 1491 int scalingMode, Rect crop) { 1492 1493 return nativeCreateImagePlanes(numPlanes, buffer, fenceFd, format, crop.left, crop.top, 1494 crop.right, crop.bottom); 1495 } 1496 nativeCreateImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, int cropRight, int cropBottom)1497 private synchronized static native ImagePlane[] nativeCreateImagePlanes(int numPlanes, 1498 GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, 1499 int cropRight, int cropBottom); 1500 1501 /** 1502 * @hide 1503 */ unlockGraphicBuffer(GraphicBuffer buffer)1504 public static void unlockGraphicBuffer(GraphicBuffer buffer) { 1505 nativeUnlockGraphicBuffer(buffer); 1506 } 1507 nativeUnlockGraphicBuffer(GraphicBuffer buffer)1508 private synchronized static native void nativeUnlockGraphicBuffer(GraphicBuffer buffer); 1509 1510 /** 1511 * We use a class initializer to allow the native code to cache some 1512 * field offsets. 1513 */ nativeClassInit()1514 private static native void nativeClassInit(); 1515 static { 1516 System.loadLibrary("media_jni"); nativeClassInit()1517 nativeClassInit(); 1518 } 1519 } 1520