1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.annotation.SuppressLint; 22 import android.graphics.GraphicBuffer; 23 import android.graphics.ImageFormat; 24 import android.graphics.ImageFormat.Format; 25 import android.graphics.Rect; 26 import android.hardware.DataSpace; 27 import android.hardware.DataSpace.NamedDataSpace; 28 import android.hardware.HardwareBuffer; 29 import android.hardware.HardwareBuffer.Usage; 30 import android.hardware.SyncFence; 31 import android.hardware.camera2.MultiResolutionImageReader; 32 import android.os.Handler; 33 import android.os.Looper; 34 import android.os.ParcelFileDescriptor; 35 import android.view.Surface; 36 37 import dalvik.system.VMRuntime; 38 39 import java.io.IOException; 40 import java.lang.ref.WeakReference; 41 import java.nio.ByteBuffer; 42 import java.nio.ByteOrder; 43 import java.nio.NioUtils; 44 import java.util.List; 45 import java.util.Objects; 46 import java.util.concurrent.CopyOnWriteArrayList; 47 import java.util.concurrent.Executor; 48 import java.util.concurrent.atomic.AtomicBoolean; 49 50 /** 51 * <p>The ImageReader class allows direct application access to image data 52 * rendered into a {@link android.view.Surface}</p> 53 * 54 * <p>Several Android media API classes accept Surface objects as targets to 55 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 56 * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and 57 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 58 * sizes and formats that can be used with each source vary, and should be 59 * checked in the documentation for the specific API.</p> 60 * 61 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 62 * objects can be accessed at the same time, up to the number specified by the 63 * {@code maxImages} constructor parameter. New images sent to an ImageReader 64 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 65 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 66 * eventually stall or drop Images in trying to render to the Surface if the 67 * ImageReader does not obtain and release Images at a rate equal to the 68 * production rate.</p> 69 */ 70 public class ImageReader implements AutoCloseable { 71 72 /** 73 * Returned by nativeImageSetup when acquiring the image was successful. 74 */ 75 private static final int ACQUIRE_SUCCESS = 0; 76 /** 77 * Returned by nativeImageSetup when we couldn't acquire the buffer, 78 * because there were no buffers available to acquire. 79 */ 80 private static final int ACQUIRE_NO_BUFS = 1; 81 /** 82 * Returned by nativeImageSetup when we couldn't acquire the buffer 83 * because the consumer has already acquired {@maxImages} and cannot 84 * acquire more than that. 85 */ 86 private static final int ACQUIRE_MAX_IMAGES = 2; 87 88 /** 89 * <p> 90 * Create a new reader for images of the desired size and format. 91 * </p> 92 * <p> 93 * The {@code maxImages} parameter determines the maximum number of 94 * {@link Image} objects that can be be acquired from the 95 * {@code ImageReader} simultaneously. Requesting more buffers will use up 96 * more memory, so it is important to use only the minimum number necessary 97 * for the use case. 98 * </p> 99 * <p> 100 * The valid sizes and formats depend on the source of the image data. 101 * </p> 102 * <p> 103 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created 104 * {@link ImageReader} will produce images that are not directly accessible 105 * by the application. The application can still acquire images from this 106 * {@link ImageReader}, and send them to the 107 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via 108 * {@link ImageWriter} interface. However, the {@link Image#getPlanes() 109 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE 110 * PRIVATE} format images. The application can check if an existing reader's 111 * format by calling {@link #getImageFormat()}. 112 * </p> 113 * <p> 114 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader 115 * ImageReaders} are more efficient to use when application access to image 116 * data is not necessary, compared to ImageReaders using other format such 117 * as {@link ImageFormat#YUV_420_888 YUV_420_888}. 118 * </p> 119 * 120 * @param width The default width in pixels of the Images that this reader 121 * will produce. 122 * @param height The default height in pixels of the Images that this reader 123 * will produce. 124 * @param format The format of the Image that this reader will produce. This 125 * must be one of the {@link android.graphics.ImageFormat} or 126 * {@link android.graphics.PixelFormat} constants. Note that not 127 * all formats are supported, like ImageFormat.NV21. 128 * @param maxImages The maximum number of images the user will want to 129 * access simultaneously. This should be as small as possible to 130 * limit memory use. Once maxImages Images are obtained by the 131 * user, one of them has to be released before a new Image will 132 * become available for access through 133 * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 134 * Must be greater than 0. 135 * @see Image 136 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages)137 public static @NonNull ImageReader newInstance( 138 @IntRange(from = 1) int width, 139 @IntRange(from = 1) int height, 140 @Format int format, 141 @IntRange(from = 1) int maxImages) { 142 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 143 // work, and is inscrutable anyway 144 return new ImageReader(width, height, format, maxImages, 145 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, null); 146 } 147 148 /** 149 * <p> 150 * Create a new reader for images of the desired size, format and consumer usage flag. 151 * </p> 152 * <p> 153 * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that 154 * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will 155 * use up more memory, so it is important to use only the minimum number necessary for the use 156 * case. 157 * </p> 158 * <p> 159 * The valid sizes and formats depend on the source of the image data. 160 * </p> 161 * <p> 162 * The format and usage flag combination describes how the buffer will be used by 163 * consumer end-points. For example, if the application intends to send the images to 164 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video 165 * encoding, the format and usage flag combination needs to be 166 * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE_VIDEO_ENCODE}. When an 167 * {@link ImageReader} object is created with a valid size and such format/usage flag 168 * combination, the application can send the {@link Image images} to an {@link ImageWriter} that 169 * is created with the input {@link android.view.Surface} provided by the 170 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. 171 * </p> 172 * <p> 173 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} 174 * will produce images that are not directly accessible by the application. The application can 175 * still acquire images from this {@link ImageReader}, and send them to the 176 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the 177 * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video 178 * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() 179 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format 180 * images. The application can check if an existing reader's format by calling 181 * {@link #getImageFormat()}. 182 * </p> 183 * <p> 184 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more 185 * efficient to use when application access to image data is not necessary, compared to 186 * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. 187 * </p> 188 * <p> 189 * Note that not all format and usage flag combinations are supported by the 190 * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} 191 * (assuming the consumer end-points support the such image consumption, e.g., hardware video 192 * encoding). 193 * <table> 194 * <tr> 195 * <th>Format</th> 196 * <th>Compatible usage flags</th> 197 * </tr> 198 * <tr> 199 * <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by 200 * {@link android.graphics.ImageFormat ImageFormat} or 201 * {@link android.graphics.PixelFormat PixelFormat}</td> 202 * <td>{@link HardwareBuffer#USAGE_CPU_READ_RARELY} or 203 * {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}</td> 204 * </tr> 205 * <tr> 206 * <td>{@link android.graphics.ImageFormat#PRIVATE}</td> 207 * <td>{@link HardwareBuffer#USAGE_VIDEO_ENCODE} or 208 * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}, or combined</td> 209 * </tr> 210 * </table> 211 * Using other combinations may result in {@link IllegalArgumentException}. Additionally, 212 * specifying {@link HardwareBuffer#USAGE_CPU_WRITE_RARELY} or 213 * {@link HardwareBuffer#USAGE_CPU_WRITE_OFTEN} and writing to the ImageReader's buffers 214 * might break assumptions made by some producers, and should be used with caution. 215 * </p> 216 * <p> 217 * If the {@link ImageReader} is used as an output target for a {@link 218 * android.hardware.camera2.CameraDevice}, and if the usage flag contains 219 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE}, the timestamps of the 220 * {@link Image images} produced by the {@link ImageReader} won't be in the same timebase as 221 * {@link android.os.SystemClock#elapsedRealtimeNanos}, even if 222 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is 223 * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME}. 224 * Instead, the timestamps will be roughly in the same timebase as in 225 * {@link android.os.SystemClock#uptimeMillis}, so that A/V synchronization could work for 226 * video recording. In this case, the timestamps from the {@link ImageReader} with 227 * {@link HardwareBuffer#USAGE_VIDEO_ENCODE} usage flag may not be directly comparable with 228 * timestamps of other streams or capture result metadata. 229 * </p> 230 * @param width The default width in pixels of the Images that this reader will produce. 231 * @param height The default height in pixels of the Images that this reader will produce. 232 * @param format The format of the Image that this reader will produce. This must be one of the 233 * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} 234 * constants. Note that not all formats are supported, like ImageFormat.NV21. 235 * @param maxImages The maximum number of images the user will want to access simultaneously. 236 * This should be as small as possible to limit memory use. Once maxImages Images are 237 * obtained by the user, one of them has to be released before a new Image will 238 * become available for access through {@link #acquireLatestImage()} or 239 * {@link #acquireNextImage()}. Must be greater than 0. 240 * @param usage The intended usage of the images produced by this ImageReader. See the usages 241 * on {@link HardwareBuffer} for a list of valid usage bits. See also 242 * {@link HardwareBuffer#isSupported(int, int, int, int, long)} for checking 243 * if a combination is supported. If it's not supported this will throw 244 * an {@link IllegalArgumentException}. 245 * @see Image 246 * @see HardwareBuffer 247 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @Usage long usage)248 public static @NonNull ImageReader newInstance( 249 @IntRange(from = 1) int width, 250 @IntRange(from = 1) int height, 251 @Format int format, 252 @IntRange(from = 1) int maxImages, 253 @Usage long usage) { 254 // TODO: Check this - can't do it just yet because format support is different 255 // Unify formats! The only reliable way to validate usage is to just try it and see. 256 257 // if (!HardwareBuffer.isSupported(width, height, format, 1, usage)) { 258 // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) 259 // + " & usage=" + Long.toHexString(usage) + " is not supported"); 260 // } 261 return new ImageReader(width, height, format, maxImages, usage, /*parent*/ null); 262 } 263 264 /** 265 * @hide 266 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @NonNull MultiResolutionImageReader parent)267 public static @NonNull ImageReader newInstance( 268 @IntRange(from = 1) int width, 269 @IntRange(from = 1) int height, 270 @Format int format, 271 @IntRange(from = 1) int maxImages, 272 @NonNull MultiResolutionImageReader parent) { 273 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 274 // work, and is inscrutable anyway 275 return new ImageReader(width, height, format, maxImages, 276 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, parent); 277 } 278 initializeImageReader(int width, int height, int imageFormat, int maxImages, long usage, int hardwareBufferFormat, int dataSpace, boolean useLegacyImageFormat)279 private void initializeImageReader(int width, int height, int imageFormat, int maxImages, 280 long usage, int hardwareBufferFormat, int dataSpace, boolean useLegacyImageFormat) { 281 if (width < 1 || height < 1) { 282 throw new IllegalArgumentException( 283 "The image dimensions must be positive"); 284 } 285 286 if (maxImages < 1) { 287 throw new IllegalArgumentException( 288 "Maximum outstanding image count must be at least 1"); 289 } 290 291 if (imageFormat == ImageFormat.NV21) { 292 throw new IllegalArgumentException( 293 "NV21 format is not supported"); 294 } 295 296 nativeInit(new WeakReference<>(this), width, height, maxImages, usage, 297 hardwareBufferFormat, dataSpace); 298 299 mIsReaderValid = true; 300 301 mSurface = nativeGetSurface(); 302 // Estimate the native buffer allocation size and register it so it gets accounted for 303 // during GC. Note that this doesn't include the buffers required by the buffer queue 304 // itself and the buffers requested by the producer. 305 // Only include memory for 1 buffer, since actually accounting for the memory used is 306 // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some 307 // size. 308 mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( 309 width, height, useLegacyImageFormat ? imageFormat : hardwareBufferFormat, 310 /*buffer count*/ 1); 311 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 312 } 313 ImageReader(int width, int height, int imageFormat, int maxImages, long usage, MultiResolutionImageReader parent)314 private ImageReader(int width, int height, int imageFormat, int maxImages, long usage, 315 MultiResolutionImageReader parent) { 316 mWidth = width; 317 mHeight = height; 318 mFormat = imageFormat; 319 mUsage = usage; 320 mMaxImages = maxImages; 321 mParent = parent; 322 // retrieve hal Format and hal dataspace from imageFormat 323 mHardwareBufferFormat = PublicFormatUtils.getHalFormat(mFormat); 324 mDataSpace = PublicFormatUtils.getHalDataspace(mFormat); 325 mUseLegacyImageFormat = true; 326 mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); 327 328 initializeImageReader(width, height, imageFormat, maxImages, usage, mHardwareBufferFormat, 329 mDataSpace, mUseLegacyImageFormat); 330 } 331 ImageReader(int width, int height, int maxImages, long usage, MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace)332 private ImageReader(int width, int height, int maxImages, long usage, 333 MultiResolutionImageReader parent, int hardwareBufferFormat, int dataSpace) { 334 mWidth = width; 335 mHeight = height; 336 mFormat = ImageFormat.UNKNOWN; // set default image format value as UNKNOWN 337 mUsage = usage; 338 mMaxImages = maxImages; 339 mParent = parent; 340 mHardwareBufferFormat = hardwareBufferFormat; 341 mDataSpace = dataSpace; 342 mUseLegacyImageFormat = false; 343 mNumPlanes = ImageUtils.getNumPlanesForHardwareBufferFormat(mHardwareBufferFormat); 344 345 initializeImageReader(width, height, mFormat, maxImages, usage, hardwareBufferFormat, 346 dataSpace, mUseLegacyImageFormat); 347 } 348 349 /** 350 * The default width of {@link Image Images}, in pixels. 351 * 352 * <p>The width may be overridden by the producer sending buffers to this 353 * ImageReader's Surface. If so, the actual width of the images can be 354 * found using {@link Image#getWidth}.</p> 355 * 356 * @return the expected width of an Image 357 */ getWidth()358 public int getWidth() { 359 return mWidth; 360 } 361 362 /** 363 * The default height of {@link Image Images}, in pixels. 364 * 365 * <p>The height may be overridden by the producer sending buffers to this 366 * ImageReader's Surface. If so, the actual height of the images can be 367 * found using {@link Image#getHeight}.</p> 368 * 369 * @return the expected height of an Image 370 */ getHeight()371 public int getHeight() { 372 return mHeight; 373 } 374 375 /** 376 * The default {@link ImageFormat image format} of {@link Image Images}. 377 * 378 * <p>Some color formats may be overridden by the producer sending buffers to 379 * this ImageReader's Surface if the default color format allows. ImageReader 380 * guarantees that all {@link Image Images} acquired from ImageReader 381 * (for example, with {@link #acquireNextImage}) will have a "compatible" 382 * format to what was specified in {@link #newInstance}. 383 * As of now, each format is only compatible to itself. 384 * The actual format of the images can be found using {@link Image#getFormat}.</p> 385 * 386 * <p>Use this function if the ImageReader instance is created by factory method 387 * {@code newInstance} function or by builder pattern {@code ImageReader.Builder} and using 388 * {@link Builder#setImageFormat}.</p> 389 * 390 * @return the expected format of an Image 391 * 392 * @see ImageFormat 393 */ getImageFormat()394 public int getImageFormat() { 395 return mFormat; 396 } 397 398 /** 399 * The default {@link HardwareBuffer} format of {@link Image Images}. 400 * 401 * <p>Use this function if the ImageReader instance is created by builder pattern 402 * {@code ImageReader.Builder} and using {@link Builder#setDefaultHardwareBufferFormat} and 403 * {@link Builder#setDefaultDataSpace}.</p> 404 * 405 * @return the expected {@link HardwareBuffer} format of an Image. 406 */ getHardwareBufferFormat()407 public @HardwareBuffer.Format int getHardwareBufferFormat() { 408 return mHardwareBufferFormat; 409 } 410 411 /** 412 * The default dataspace of {@link Image Images}. 413 * 414 * <p>Use this function if the ImageReader instance is created by builder pattern 415 * {@code ImageReader.Builder} and {@link Builder#setDefaultDataSpace}.</p> 416 * 417 * @return the expected dataspace of an Image. 418 */ 419 @SuppressLint("MethodNameUnits") getDataSpace()420 public @NamedDataSpace int getDataSpace() { 421 return mDataSpace; 422 } 423 424 /** 425 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 426 * with {@link #acquireNextImage}). 427 * 428 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 429 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 430 * </p> 431 * 432 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 433 * acquire function throwing a {@link IllegalStateException}. Furthermore, 434 * while the max number of images have been acquired by the ImageReader user, the producer 435 * enqueueing additional images may stall until at least one image has been released. </p> 436 * 437 * @return Maximum number of images for this ImageReader. 438 * 439 * @see Image#close 440 */ getMaxImages()441 public int getMaxImages() { 442 return mMaxImages; 443 } 444 445 /** 446 * The usage flag of images that can be produced by the ImageReader. 447 * 448 * @return The usage flag of the images for this ImageReader. 449 */ getUsage()450 public @Usage long getUsage() { 451 return mUsage; 452 } 453 454 /** 455 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 456 * {@code ImageReader}.</p> 457 * 458 * <p>Until valid image data is rendered into this {@link Surface}, the 459 * {@link #acquireNextImage} method will return {@code null}. Only one source 460 * can be producing data into this Surface at the same time, although the 461 * same {@link Surface} can be reused with a different API once the first source is 462 * disconnected from the {@link Surface}.</p> 463 * 464 * <p>Please note that holding on to the Surface object returned by this method is not enough 465 * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a 466 * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.</p> 467 * 468 * @return A {@link Surface} to use for a drawing target for various APIs. 469 */ getSurface()470 public Surface getSurface() { 471 return mSurface; 472 } 473 474 /** 475 * <p> 476 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 477 * {@link Image images}. Returns {@code null} if no new image is available. 478 * </p> 479 * <p> 480 * This operation will acquire all the images possible from the ImageReader, 481 * but {@link #close} all images that aren't the latest. This function is 482 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 483 * more suited for real-time processing. 484 * </p> 485 * <p> 486 * Note that {@link #getMaxImages maxImages} should be at least 2 for 487 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 488 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 489 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 490 * with less than two images of margin, that is 491 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 492 * </p> 493 * <p> 494 * This operation will fail by throwing an {@link IllegalStateException} if 495 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 496 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 497 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 498 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 499 * will be thrown until more images are 500 * released with {@link Image#close}. 501 * </p> 502 * 503 * @return latest frame of image data, or {@code null} if no image data is available. 504 * @throws IllegalStateException if too many images are currently acquired 505 */ acquireLatestImage()506 public Image acquireLatestImage() { 507 Image image = acquireNextImage(); 508 if (image == null) { 509 return null; 510 } 511 try { 512 for (;;) { 513 Image next = acquireNextImageNoThrowISE(); 514 if (next == null) { 515 Image result = image; 516 image = null; 517 return result; 518 } 519 image.close(); 520 image = next; 521 } 522 } finally { 523 if (image != null) { 524 image.close(); 525 } 526 if (mParent != null) { 527 mParent.flushOther(this); 528 } 529 } 530 } 531 532 /** 533 * Don't throw IllegalStateException if there are too many images acquired. 534 * 535 * @return Image if acquiring succeeded, or null otherwise. 536 * 537 * @hide 538 */ acquireNextImageNoThrowISE()539 public Image acquireNextImageNoThrowISE() { 540 SurfaceImage si; 541 if (mUseLegacyImageFormat) { 542 si = new SurfaceImage(mFormat); 543 } else { 544 si = new SurfaceImage(mHardwareBufferFormat, mDataSpace); 545 } 546 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 547 } 548 549 /** 550 * Attempts to acquire the next image from the underlying native implementation. 551 * 552 * <p> 553 * Note that unexpected failures will throw at the JNI level. 554 * </p> 555 * 556 * @param si A blank SurfaceImage. 557 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 558 * 559 * @see #ACQUIRE_MAX_IMAGES 560 * @see #ACQUIRE_NO_BUFS 561 * @see #ACQUIRE_SUCCESS 562 */ acquireNextSurfaceImage(SurfaceImage si)563 private int acquireNextSurfaceImage(SurfaceImage si) { 564 synchronized (mCloseLock) { 565 // A null image will eventually be returned if ImageReader is already closed. 566 int status = ACQUIRE_NO_BUFS; 567 if (mIsReaderValid) { 568 status = nativeImageSetup(si, mUseLegacyImageFormat); 569 } 570 571 switch (status) { 572 case ACQUIRE_SUCCESS: 573 si.mIsImageValid = true; 574 case ACQUIRE_NO_BUFS: 575 case ACQUIRE_MAX_IMAGES: 576 break; 577 default: 578 throw new AssertionError("Unknown nativeImageSetup return code " + status); 579 } 580 581 // Only keep track the successfully acquired image, as the native buffer is only mapped 582 // for such case. 583 if (status == ACQUIRE_SUCCESS) { 584 mAcquiredImages.add(si); 585 } 586 return status; 587 } 588 } 589 590 /** 591 * <p> 592 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 593 * no new image is available. 594 * </p> 595 * 596 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 597 * automatically release older images, and allow slower-running processing routines to catch 598 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 599 * batch/background processing. Incorrectly using this function can cause images to appear 600 * with an ever-increasing delay, followed by a complete stall where no new images seem to 601 * appear. 602 * </p> 603 * 604 * <p> 605 * This operation will fail by throwing an {@link IllegalStateException} if 606 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 607 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 608 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 609 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 610 * {@link IllegalStateException} will be thrown until more images are released with 611 * {@link Image#close}. 612 * </p> 613 * 614 * @return a new frame of image data, or {@code null} if no image data is available. 615 * @throws IllegalStateException if {@code maxImages} images are currently acquired 616 * @see #acquireLatestImage 617 */ acquireNextImage()618 public Image acquireNextImage() { 619 // Initialize with reader format, but can be overwritten by native if the image 620 // format is different from the reader format. 621 SurfaceImage si; 622 if (mUseLegacyImageFormat) { 623 si = new SurfaceImage(mFormat); 624 } else { 625 si = new SurfaceImage(mHardwareBufferFormat, mDataSpace); 626 } 627 int status = acquireNextSurfaceImage(si); 628 629 switch (status) { 630 case ACQUIRE_SUCCESS: 631 return si; 632 case ACQUIRE_NO_BUFS: 633 return null; 634 case ACQUIRE_MAX_IMAGES: 635 throw new IllegalStateException( 636 String.format( 637 "maxImages (%d) has already been acquired, " + 638 "call #close before acquiring more.", mMaxImages)); 639 default: 640 throw new AssertionError("Unknown nativeImageSetup return code " + status); 641 } 642 } 643 644 /** 645 * <p>Return the frame to the ImageReader for reuse.</p> 646 * 647 * This method should only be called via {@link SurfaceImage#close} which ensures that image 648 * closing is atomic. 649 */ releaseImage(Image i)650 private void releaseImage(Image i) { 651 if (! (i instanceof SurfaceImage) ) { 652 throw new IllegalArgumentException( 653 "This image was not produced by an ImageReader"); 654 } 655 SurfaceImage si = (SurfaceImage) i; 656 if (si.mIsImageValid == false) { 657 return; 658 } 659 660 if (si.getReader() != this || !mAcquiredImages.contains(i)) { 661 throw new IllegalArgumentException( 662 "This image was not produced by this ImageReader"); 663 } 664 665 si.clearSurfacePlanes(); 666 nativeReleaseImage(i); 667 si.mIsImageValid = false; 668 mAcquiredImages.remove(i); 669 } 670 671 /** 672 * Register a listener to be invoked when a new image becomes available 673 * from the ImageReader. 674 * 675 * @param listener 676 * The listener that will be run. 677 * @param handler 678 * The handler on which the listener should be invoked, or null 679 * if the listener should be invoked on the calling thread's looper. 680 * @throws IllegalArgumentException 681 * If no handler specified and the calling thread has no looper. 682 */ setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler)683 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 684 synchronized (mListenerLock) { 685 if (listener != null) { 686 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 687 if (looper == null) { 688 throw new IllegalArgumentException( 689 "handler is null but the current thread is not a looper"); 690 } 691 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 692 mListenerHandler = new ListenerHandler(looper); 693 mListenerExecutor = new HandlerExecutor(mListenerHandler); 694 } 695 } else { 696 mListenerHandler = null; 697 mListenerExecutor = null; 698 } 699 mListener = listener; 700 } 701 } 702 703 /** 704 * Register a listener to be invoked when a new image becomes available 705 * from the ImageReader. 706 * 707 * @param listener 708 * The listener that will be run. 709 * @param executor 710 * The executor which will be used to invoke the listener. 711 * @throws IllegalArgumentException 712 * If no handler specified and the calling thread has no looper. 713 * 714 * @hide 715 */ setOnImageAvailableListenerWithExecutor(@onNull OnImageAvailableListener listener, @NonNull Executor executor)716 public void setOnImageAvailableListenerWithExecutor(@NonNull OnImageAvailableListener listener, 717 @NonNull Executor executor) { 718 if (executor == null) { 719 throw new IllegalArgumentException("executor must not be null"); 720 } 721 722 synchronized (mListenerLock) { 723 mListenerExecutor = executor; 724 mListener = listener; 725 } 726 } 727 728 /** 729 * Callback interface for being notified that a new image is available. 730 * 731 * <p> 732 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 733 * available from ImageReader. 734 * </p> 735 */ 736 public interface OnImageAvailableListener { 737 /** 738 * Callback that is called when a new image is available from ImageReader. 739 * 740 * @param reader the ImageReader the callback is associated with. 741 * @see ImageReader 742 * @see Image 743 */ onImageAvailable(ImageReader reader)744 void onImageAvailable(ImageReader reader); 745 } 746 747 /** 748 * Free up all the resources associated with this ImageReader. 749 * 750 * <p> 751 * After calling this method, this ImageReader can not be used. Calling 752 * any methods on this ImageReader and Images previously provided by 753 * {@link #acquireNextImage} or {@link #acquireLatestImage} 754 * will result in an {@link IllegalStateException}, and attempting to read from 755 * {@link ByteBuffer ByteBuffers} returned by an earlier 756 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 757 * have undefined behavior. 758 * </p> 759 */ 760 @Override close()761 public void close() { 762 setOnImageAvailableListener(null, null); 763 if (mSurface != null) mSurface.release(); 764 765 /** 766 * Close all outstanding acquired images before closing the ImageReader. It is a good 767 * practice to close all the images as soon as it is not used to reduce system instantaneous 768 * memory pressure. CopyOnWrite list will use a copy of current list content. For the images 769 * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For 770 * the image being acquired by other threads, mCloseLock is used to synchronize close and 771 * acquire operations. 772 */ 773 synchronized (mCloseLock) { 774 mIsReaderValid = false; 775 for (Image image : mAcquiredImages) { 776 image.close(); 777 } 778 mAcquiredImages.clear(); 779 780 nativeClose(); 781 782 if (mEstimatedNativeAllocBytes > 0) { 783 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 784 mEstimatedNativeAllocBytes = 0; 785 } 786 } 787 } 788 789 /** 790 * Discard any free buffers owned by this ImageReader. 791 * 792 * <p> 793 * Generally, the ImageReader caches buffers for reuse once they have been 794 * allocated, for best performance. However, sometimes it may be important to 795 * release all the cached, unused buffers to save on memory. 796 * </p> 797 * <p> 798 * Calling this method will discard all free cached buffers. This does not include any buffers 799 * associated with Images acquired from the ImageReader, any filled buffers waiting to be 800 * acquired, and any buffers currently in use by the source rendering buffers into the 801 * ImageReader's Surface. 802 * <p> 803 * The ImageReader continues to be usable after this call, but may need to reallocate buffers 804 * when more buffers are needed for rendering. 805 * </p> 806 */ discardFreeBuffers()807 public void discardFreeBuffers() { 808 synchronized (mCloseLock) { 809 nativeDiscardFreeBuffers(); 810 } 811 } 812 813 @Override finalize()814 protected void finalize() throws Throwable { 815 try { 816 close(); 817 } finally { 818 super.finalize(); 819 } 820 } 821 822 /** 823 * <p> 824 * Remove the ownership of this image from the ImageReader. 825 * </p> 826 * <p> 827 * After this call, the ImageReader no longer owns this image, and the image 828 * ownership can be transfered to another entity like {@link ImageWriter} 829 * via {@link ImageWriter#queueInputImage}. It's up to the new owner to 830 * release the resources held by this image. For example, if the ownership 831 * of this image is transfered to an {@link ImageWriter}, the image will be 832 * freed by the ImageWriter after the image data consumption is done. 833 * </p> 834 * <p> 835 * This method can be used to achieve zero buffer copy for use cases like 836 * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV 837 * reprocessing, where the application can select an output image from 838 * {@link ImageReader} and transfer this image directly to 839 * {@link ImageWriter}, where this image can be consumed by camera directly. 840 * For PRIVATE reprocessing, this is the only way to send input buffers to 841 * the {@link android.hardware.camera2.CameraDevice camera} for 842 * reprocessing. 843 * </p> 844 * <p> 845 * This is a package private method that is only used internally. 846 * </p> 847 * 848 * @param image The image to be detached from this ImageReader. 849 * @throws IllegalStateException If the ImageReader or image have been 850 * closed, or the has been detached, or has not yet been 851 * acquired. 852 * @hide 853 */ detachImage(Image image)854 public void detachImage(Image image) { 855 if (image == null) { 856 throw new IllegalArgumentException("input image must not be null"); 857 } 858 if (!isImageOwnedbyMe(image)) { 859 throw new IllegalArgumentException("Trying to detach an image that is not owned by" 860 + " this ImageReader"); 861 } 862 863 SurfaceImage si = (SurfaceImage) image; 864 si.throwISEIfImageIsInvalid(); 865 866 if (si.isAttachable()) { 867 throw new IllegalStateException("Image was already detached from this ImageReader"); 868 } 869 870 nativeDetachImage(image); 871 si.clearSurfacePlanes(); 872 si.mPlanes = null; 873 si.setDetached(true); 874 } 875 isImageOwnedbyMe(Image image)876 private boolean isImageOwnedbyMe(Image image) { 877 if (!(image instanceof SurfaceImage)) { 878 return false; 879 } 880 SurfaceImage si = (SurfaceImage) image; 881 return si.getReader() == this; 882 } 883 884 /** 885 * Called from Native code when an Event happens. 886 * 887 * This may be called from an arbitrary Binder thread, so access to the ImageReader must be 888 * synchronized appropriately. 889 */ postEventFromNative(Object selfRef)890 private static void postEventFromNative(Object selfRef) { 891 @SuppressWarnings("unchecked") 892 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 893 final ImageReader ir = weakSelf.get(); 894 if (ir == null) { 895 return; 896 } 897 898 final Executor executor; 899 final OnImageAvailableListener listener; 900 synchronized (ir.mListenerLock) { 901 executor = ir.mListenerExecutor; 902 listener = ir.mListener; 903 } 904 final boolean isReaderValid; 905 synchronized (ir.mCloseLock) { 906 isReaderValid = ir.mIsReaderValid; 907 } 908 909 // It's dangerous to fire onImageAvailable() callback when the ImageReader 910 // is being closed, as application could acquire next image in the 911 // onImageAvailable() callback. 912 if (executor != null && listener != null && isReaderValid) { 913 executor.execute(new Runnable() { 914 @Override 915 public void run() { 916 listener.onImageAvailable(ir); 917 } 918 }); 919 } 920 } 921 922 /** 923 * Builder class for {@link ImageReader} objects. 924 */ 925 public static final class Builder { 926 private int mWidth; 927 private int mHeight; 928 private int mMaxImages = 1; 929 private int mImageFormat = ImageFormat.UNKNOWN; 930 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 931 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 932 private long mUsage = HardwareBuffer.USAGE_CPU_READ_OFTEN; 933 private boolean mUseLegacyImageFormat = false; 934 935 /** 936 * Constructs a new builder for {@link ImageReader}. 937 * 938 * @param width The default width in pixels that will be passed to the producer. 939 * May be overridden by the producer. 940 * @param height The default height in pixels that will be passed to the producer. 941 * May be overridden by the producer. 942 * @see Image 943 */ Builder(@ntRangefrom = 1) int width, @IntRange(from = 1) int height)944 public Builder(@IntRange(from = 1) int width, @IntRange(from = 1) int height) { 945 mWidth = width; 946 mHeight = height; 947 } 948 949 /** 950 * Set the maximal number of images. 951 * 952 * @param maxImages The maximum number of images the user will want to 953 * access simultaneously. This should be as small as possible to 954 * limit memory use. Default value is 1. 955 * @return the Builder instance with customized usage value. 956 */ setMaxImages(int maxImages)957 public @NonNull Builder setMaxImages(int maxImages) { 958 mMaxImages = maxImages; 959 return this; 960 } 961 962 /** 963 * Set the consumer usage flag. 964 * 965 * @param usage The intended usage of the images consumed by this ImageReader. 966 * See the usages on {@link HardwareBuffer} for a list of valid usage bits. 967 * Default value is {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}. 968 * @return the Builder instance with customized usage value. 969 * 970 * @see HardwareBuffer 971 */ setUsage(@sage long usage)972 public @NonNull Builder setUsage(@Usage long usage) { 973 mUsage = usage; 974 return this; 975 } 976 977 /** 978 * Set the default image format passed by the producer. May be overridden by the producer. 979 * 980 * <p>{@link #setImageFormat} function replaces the combination of 981 * {@link #setDefaultHardwareBufferFormat} and {@link #setDefaultDataSpace} functions. 982 * Either this or these two functions must be called to initialize an {@code ImageReader} 983 * instance.</p> 984 * 985 * @param imageFormat The format of the image that this reader will produce. This 986 * must be one of the {@link android.graphics.ImageFormat} or 987 * {@link android.graphics.PixelFormat} constants. Note that not 988 * all formats are supported, like ImageFormat.NV21. The default value is 989 * {@link ImageFormat#UNKNOWN}. 990 * @return the builder instance with customized image format value. 991 * 992 * @see #setDefaultHardwareBufferFormat 993 * @see #setDefaultDataSpace 994 */ setImageFormat(@ormat int imageFormat)995 public @NonNull Builder setImageFormat(@Format int imageFormat) { 996 mImageFormat = imageFormat; 997 mUseLegacyImageFormat = true; 998 mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 999 mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1000 return this; 1001 } 1002 1003 /** 1004 * Set the default hardwareBuffer format passed by the producer. 1005 * May be overridden by the producer. 1006 * 1007 * <p>This function works together with {@link #setDefaultDataSpace} for an 1008 * {@link ImageReader} instance. Setting at least one of these two replaces 1009 * {@link #setImageFormat} function.</p> 1010 * 1011 * <p>The format of the Image can be overridden after {@link #setImageFormat} by calling 1012 * this function and then {@link #setDefaultDataSpace} functions. 1013 * <i>Warning:</i> Missing one of callings for initializing or overriding the format may 1014 * involve undefined behaviors.</p> 1015 * 1016 * @param hardwareBufferFormat The HardwareBuffer format of the image that this reader 1017 * will produce. The default value is 1018 * {@link HardwareBuffer#RGBA_8888 HardwareBuffer.RGBA_8888}. 1019 * @return the builder instance with customized hardwareBuffer value. 1020 * 1021 * @see #setDefaultDataSpace 1022 * @see #setImageFormat 1023 */ 1024 @SuppressLint("MissingGetterMatchingBuilder") setDefaultHardwareBufferFormat( @ardwareBuffer.Format int hardwareBufferFormat)1025 public @NonNull Builder setDefaultHardwareBufferFormat( 1026 @HardwareBuffer.Format int hardwareBufferFormat) { 1027 mHardwareBufferFormat = hardwareBufferFormat; 1028 mUseLegacyImageFormat = false; 1029 mImageFormat = ImageFormat.UNKNOWN; 1030 return this; 1031 } 1032 1033 /** 1034 * Set the default dataspace passed by the producer. 1035 * May be overridden by the producer. 1036 * 1037 * <p>This function works together with {@link #setDefaultHardwareBufferFormat} for an 1038 * {@link ImageReader} instance. Setting at least one of these two replaces 1039 * {@link #setImageFormat} function.</p> 1040 * 1041 * @param dataSpace The dataspace of the image that this reader will produce. 1042 * The default value is {@link DataSpace#DATASPACE_UNKNOWN}. 1043 * @return the builder instance with customized dataspace value. 1044 * 1045 * @see #setDefaultHardwareBufferFormat 1046 */ 1047 @SuppressLint("MissingGetterMatchingBuilder") setDefaultDataSpace(@amedDataSpace int dataSpace)1048 public @NonNull Builder setDefaultDataSpace(@NamedDataSpace int dataSpace) { 1049 mDataSpace = dataSpace; 1050 mUseLegacyImageFormat = false; 1051 mImageFormat = ImageFormat.UNKNOWN; 1052 return this; 1053 } 1054 1055 /** 1056 * Builds a new ImageReader object. 1057 * 1058 * @return The new ImageReader object. 1059 */ build()1060 public @NonNull ImageReader build() { 1061 if (mUseLegacyImageFormat) { 1062 return new ImageReader(mWidth, mHeight, mImageFormat, mMaxImages, mUsage, null); 1063 } else { 1064 return new ImageReader(mWidth, mHeight, mMaxImages, mUsage, null, 1065 mHardwareBufferFormat, mDataSpace); 1066 } 1067 } 1068 } 1069 1070 private final int mWidth; 1071 private final int mHeight; 1072 private final int mFormat; 1073 private final long mUsage; 1074 private final int mMaxImages; 1075 private final int mNumPlanes; 1076 private Surface mSurface; 1077 private int mEstimatedNativeAllocBytes; 1078 1079 private final Object mListenerLock = new Object(); 1080 private final Object mCloseLock = new Object(); 1081 private boolean mIsReaderValid = false; 1082 private OnImageAvailableListener mListener; 1083 private Executor mListenerExecutor; 1084 private ListenerHandler mListenerHandler; 1085 // Keep track of the successfully acquired Images. This need to be thread safe as the images 1086 // could be closed by different threads (e.g., application thread and GC thread). 1087 private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>(); 1088 1089 // Applicable if this isn't a standalone ImageReader, but belongs to a 1090 // MultiResolutionImageReader. 1091 private final MultiResolutionImageReader mParent; 1092 1093 private final int mHardwareBufferFormat; 1094 1095 private final @NamedDataSpace int mDataSpace; 1096 1097 private final boolean mUseLegacyImageFormat; 1098 1099 /** 1100 * This field is used by native code, do not access or modify. 1101 */ 1102 private long mNativeContext; 1103 1104 /** 1105 * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. 1106 */ 1107 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)1108 public ListenerHandler(Looper looper) { 1109 super(looper, null, true /*async*/); 1110 } 1111 } 1112 1113 /** 1114 * An adapter {@link Executor} that posts all executed tasks onto the 1115 * given {@link Handler}. 1116 **/ 1117 private final class HandlerExecutor implements Executor { 1118 private final Handler mHandler; 1119 HandlerExecutor(@onNull Handler handler)1120 public HandlerExecutor(@NonNull Handler handler) { 1121 mHandler = Objects.requireNonNull(handler); 1122 } 1123 1124 @Override execute(Runnable command)1125 public void execute(Runnable command) { 1126 mHandler.post(command); 1127 } 1128 } 1129 1130 private class SurfaceImage extends android.media.Image { 1131 private final Object mCloseLock = new Object(); 1132 SurfaceImage(int format)1133 public SurfaceImage(int format) { 1134 mFormat = format; 1135 mHardwareBufferFormat = ImageReader.this.mHardwareBufferFormat; 1136 mDataSpace = ImageReader.this.mDataSpace; 1137 } 1138 SurfaceImage(int hardwareBufferFormat, int dataSpace)1139 SurfaceImage(int hardwareBufferFormat, int dataSpace) { 1140 mHardwareBufferFormat = hardwareBufferFormat; 1141 mDataSpace = dataSpace; 1142 mFormat = PublicFormatUtils.getPublicFormat(mHardwareBufferFormat, mDataSpace); 1143 } 1144 1145 @Override close()1146 public void close() { 1147 synchronized (this.mCloseLock) { 1148 ImageReader.this.releaseImage(this); 1149 } 1150 } 1151 getReader()1152 public ImageReader getReader() { 1153 return ImageReader.this; 1154 } 1155 1156 @Override getFormat()1157 public int getFormat() { 1158 throwISEIfImageIsInvalid(); 1159 // update mFormat only if ImageReader is initialized by factory pattern. 1160 // if using builder pattern, mFormat has been updated upon initialization. 1161 // no need update here. 1162 if (ImageReader.this.mUseLegacyImageFormat) { 1163 int readerFormat = ImageReader.this.getImageFormat(); 1164 // Assume opaque reader always produce opaque images. 1165 mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : 1166 nativeGetFormat(readerFormat); 1167 } 1168 return mFormat; 1169 } 1170 1171 @Override getWidth()1172 public int getWidth() { 1173 throwISEIfImageIsInvalid(); 1174 int width; 1175 switch(getFormat()) { 1176 case ImageFormat.JPEG: 1177 case ImageFormat.DEPTH_POINT_CLOUD: 1178 case ImageFormat.RAW_PRIVATE: 1179 case ImageFormat.DEPTH_JPEG: 1180 case ImageFormat.HEIC: 1181 width = ImageReader.this.getWidth(); 1182 break; 1183 default: 1184 width = nativeGetWidth(); 1185 } 1186 return width; 1187 } 1188 1189 @Override getHeight()1190 public int getHeight() { 1191 throwISEIfImageIsInvalid(); 1192 int height; 1193 switch(getFormat()) { 1194 case ImageFormat.JPEG: 1195 case ImageFormat.DEPTH_POINT_CLOUD: 1196 case ImageFormat.RAW_PRIVATE: 1197 case ImageFormat.DEPTH_JPEG: 1198 case ImageFormat.HEIC: 1199 height = ImageReader.this.getHeight(); 1200 break; 1201 default: 1202 height = nativeGetHeight(); 1203 } 1204 return height; 1205 } 1206 1207 @Override getTimestamp()1208 public long getTimestamp() { 1209 throwISEIfImageIsInvalid(); 1210 return mTimestamp; 1211 } 1212 1213 @Override getTransform()1214 public int getTransform() { 1215 throwISEIfImageIsInvalid(); 1216 return mTransform; 1217 } 1218 1219 @Override getScalingMode()1220 public int getScalingMode() { 1221 throwISEIfImageIsInvalid(); 1222 return mScalingMode; 1223 } 1224 1225 @Override getPlaneCount()1226 public int getPlaneCount() { 1227 throwISEIfImageIsInvalid(); 1228 return ImageReader.this.mNumPlanes; 1229 } 1230 1231 @Override getFence()1232 public SyncFence getFence() throws IOException { 1233 throwISEIfImageIsInvalid(); 1234 // duplicate ParcelFileDescriptor because native still retains the fence ownership. 1235 int fence = nativeGetFenceFd(); 1236 if (fence != -1) { 1237 return SyncFence.create(ParcelFileDescriptor.fromFd(nativeGetFenceFd())); 1238 } else { 1239 return SyncFence.createEmpty(); 1240 } 1241 } 1242 1243 @Override getHardwareBuffer()1244 public HardwareBuffer getHardwareBuffer() { 1245 throwISEIfImageIsInvalid(); 1246 return nativeGetHardwareBuffer(); 1247 } 1248 1249 @Override getDataSpace()1250 public @NamedDataSpace int getDataSpace() { 1251 throwISEIfImageIsInvalid(); 1252 return mDataSpace; 1253 } 1254 1255 @Override setTimestamp(long timestampNs)1256 public void setTimestamp(long timestampNs) { 1257 throwISEIfImageIsInvalid(); 1258 mTimestamp = timestampNs; 1259 } 1260 1261 @Override getPlanes()1262 public Plane[] getPlanes() { 1263 throwISEIfImageIsInvalid(); 1264 1265 if (mPlanes == null) { 1266 mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat, 1267 ImageReader.this.mUsage); 1268 } 1269 // Shallow copy is fine. 1270 return mPlanes.clone(); 1271 } 1272 1273 @Override finalize()1274 protected final void finalize() throws Throwable { 1275 try { 1276 close(); 1277 } finally { 1278 super.finalize(); 1279 } 1280 } 1281 1282 @Override isAttachable()1283 public boolean isAttachable() { 1284 throwISEIfImageIsInvalid(); 1285 return mIsDetached.get(); 1286 } 1287 1288 @Override getOwner()1289 ImageReader getOwner() { 1290 throwISEIfImageIsInvalid(); 1291 return ImageReader.this; 1292 } 1293 1294 @Override getNativeContext()1295 long getNativeContext() { 1296 throwISEIfImageIsInvalid(); 1297 return mNativeBuffer; 1298 } 1299 setDetached(boolean detached)1300 private void setDetached(boolean detached) { 1301 throwISEIfImageIsInvalid(); 1302 mIsDetached.getAndSet(detached); 1303 } 1304 clearSurfacePlanes()1305 private void clearSurfacePlanes() { 1306 // Image#getPlanes may not be called before the image is closed. 1307 if (mIsImageValid && mPlanes != null) { 1308 for (int i = 0; i < mPlanes.length; i++) { 1309 if (mPlanes[i] != null) { 1310 mPlanes[i].clearBuffer(); 1311 mPlanes[i] = null; 1312 } 1313 } 1314 } 1315 } 1316 1317 private class SurfacePlane extends android.media.Image.Plane { 1318 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 1319 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)1320 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1321 mRowStride = rowStride; 1322 mPixelStride = pixelStride; 1323 mBuffer = buffer; 1324 /** 1325 * Set the byteBuffer order according to host endianness (native 1326 * order), otherwise, the byteBuffer order defaults to 1327 * ByteOrder.BIG_ENDIAN. 1328 */ 1329 mBuffer.order(ByteOrder.nativeOrder()); 1330 } 1331 1332 @Override getBuffer()1333 public ByteBuffer getBuffer() { 1334 throwISEIfImageIsInvalid(); 1335 return mBuffer; 1336 } 1337 1338 @Override getPixelStride()1339 public int getPixelStride() { 1340 SurfaceImage.this.throwISEIfImageIsInvalid(); 1341 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1342 throw new UnsupportedOperationException( 1343 "getPixelStride is not supported for RAW_PRIVATE plane"); 1344 } 1345 return mPixelStride; 1346 } 1347 1348 @Override getRowStride()1349 public int getRowStride() { 1350 SurfaceImage.this.throwISEIfImageIsInvalid(); 1351 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 1352 throw new UnsupportedOperationException( 1353 "getRowStride is not supported for RAW_PRIVATE plane"); 1354 } 1355 return mRowStride; 1356 } 1357 clearBuffer()1358 private void clearBuffer() { 1359 // Need null check first, as the getBuffer() may not be called before an image 1360 // is closed. 1361 if (mBuffer == null) { 1362 return; 1363 } 1364 1365 if (mBuffer.isDirect()) { 1366 NioUtils.freeDirectBuffer(mBuffer); 1367 } 1368 mBuffer = null; 1369 } 1370 1371 final private int mPixelStride; 1372 final private int mRowStride; 1373 1374 private ByteBuffer mBuffer; 1375 } 1376 1377 /** 1378 * This field is used to keep track of native object and used by native code only. 1379 * Don't modify. 1380 */ 1381 private long mNativeBuffer; 1382 1383 /** 1384 * These fields are set by native code during nativeImageSetup(). 1385 */ 1386 private long mTimestamp; 1387 private int mTransform; 1388 private int mScalingMode; 1389 1390 private SurfacePlane[] mPlanes; 1391 private int mFormat = ImageFormat.UNKNOWN; 1392 private int mHardwareBufferFormat = HardwareBuffer.RGBA_8888; 1393 private int mDataSpace = DataSpace.DATASPACE_UNKNOWN; 1394 // If this image is detached from the ImageReader. 1395 private AtomicBoolean mIsDetached = new AtomicBoolean(false); 1396 nativeCreatePlanes(int numPlanes, int readerFormat, long readerUsage)1397 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, 1398 int readerFormat, long readerUsage); nativeGetWidth()1399 private synchronized native int nativeGetWidth(); nativeGetHeight()1400 private synchronized native int nativeGetHeight(); nativeGetFormat(int readerFormat)1401 private synchronized native int nativeGetFormat(int readerFormat); nativeGetFenceFd()1402 private synchronized native int nativeGetFenceFd(); nativeGetHardwareBuffer()1403 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 1404 } 1405 nativeInit(Object weakSelf, int w, int h, int maxImgs, long consumerUsage, int hardwareBufferFormat, int dataSpace)1406 private synchronized native void nativeInit(Object weakSelf, int w, int h, int maxImgs, 1407 long consumerUsage, int hardwareBufferFormat, int dataSpace); nativeClose()1408 private synchronized native void nativeClose(); nativeReleaseImage(Image i)1409 private synchronized native void nativeReleaseImage(Image i); nativeGetSurface()1410 private synchronized native Surface nativeGetSurface(); nativeDetachImage(Image i)1411 private synchronized native int nativeDetachImage(Image i); nativeDiscardFreeBuffers()1412 private synchronized native void nativeDiscardFreeBuffers(); 1413 1414 /** 1415 * @return A return code {@code ACQUIRE_*} 1416 * 1417 * @see #ACQUIRE_SUCCESS 1418 * @see #ACQUIRE_NO_BUFS 1419 * @see #ACQUIRE_MAX_IMAGES 1420 */ nativeImageSetup(Image i, boolean legacyValidateImageFormat)1421 private synchronized native int nativeImageSetup(Image i, boolean legacyValidateImageFormat); 1422 1423 /** 1424 * @hide 1425 */ 1426 public static class ImagePlane extends android.media.Image.Plane { ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer)1427 private ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 1428 mRowStride = rowStride; 1429 mPixelStride = pixelStride; 1430 mBuffer = buffer; 1431 /** 1432 * Set the byteBuffer order according to host endianness (native 1433 * order), otherwise, the byteBuffer order defaults to 1434 * ByteOrder.BIG_ENDIAN. 1435 */ 1436 mBuffer.order(ByteOrder.nativeOrder()); 1437 } 1438 1439 @Override getBuffer()1440 public ByteBuffer getBuffer() { 1441 return mBuffer; 1442 } 1443 1444 @Override getPixelStride()1445 public int getPixelStride() { 1446 return mPixelStride; 1447 } 1448 1449 @Override getRowStride()1450 public int getRowStride() { 1451 return mRowStride; 1452 } 1453 1454 final private int mPixelStride; 1455 final private int mRowStride; 1456 1457 private ByteBuffer mBuffer; 1458 } 1459 1460 /** 1461 * @hide 1462 */ initializeImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, int scalingMode, Rect crop)1463 public static ImagePlane[] initializeImagePlanes(int numPlanes, 1464 GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, 1465 int scalingMode, Rect crop) { 1466 1467 return nativeCreateImagePlanes(numPlanes, buffer, fenceFd, format, crop.left, crop.top, 1468 crop.right, crop.bottom); 1469 } 1470 nativeCreateImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, int cropRight, int cropBottom)1471 private synchronized static native ImagePlane[] nativeCreateImagePlanes(int numPlanes, 1472 GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, 1473 int cropRight, int cropBottom); 1474 1475 /** 1476 * @hide 1477 */ unlockGraphicBuffer(GraphicBuffer buffer)1478 public static void unlockGraphicBuffer(GraphicBuffer buffer) { 1479 nativeUnlockGraphicBuffer(buffer); 1480 } 1481 nativeUnlockGraphicBuffer(GraphicBuffer buffer)1482 private synchronized static native void nativeUnlockGraphicBuffer(GraphicBuffer buffer); 1483 1484 /** 1485 * We use a class initializer to allow the native code to cache some 1486 * field offsets. 1487 */ nativeClassInit()1488 private static native void nativeClassInit(); 1489 static { 1490 System.loadLibrary("media_jni"); nativeClassInit()1491 nativeClassInit(); 1492 } 1493 } 1494