1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.graphics.ImageFormat; 20 import android.graphics.PixelFormat; 21 import android.hardware.HardwareBuffer; 22 import android.os.Handler; 23 import android.os.Looper; 24 import android.os.Message; 25 import android.util.Log; 26 import android.view.Surface; 27 28 import dalvik.system.VMRuntime; 29 30 import java.lang.ref.WeakReference; 31 import java.nio.ByteBuffer; 32 import java.nio.ByteOrder; 33 import java.nio.NioUtils; 34 import java.util.List; 35 import java.util.concurrent.CopyOnWriteArrayList; 36 import java.util.concurrent.atomic.AtomicBoolean; 37 38 /** 39 * <p>The ImageReader class allows direct application access to image data 40 * rendered into a {@link android.view.Surface}</p> 41 * 42 * <p>Several Android media API classes accept Surface objects as targets to 43 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 44 * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and 45 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 46 * sizes and formats that can be used with each source vary, and should be 47 * checked in the documentation for the specific API.</p> 48 * 49 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 50 * objects can be accessed at the same time, up to the number specified by the 51 * {@code maxImages} constructor parameter. New images sent to an ImageReader 52 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 53 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 54 * eventually stall or drop Images in trying to render to the Surface if the 55 * ImageReader does not obtain and release Images at a rate equal to the 56 * production rate.</p> 57 */ 58 public class ImageReader implements AutoCloseable { 59 60 /** 61 * Returned by nativeImageSetup when acquiring the image was successful. 62 */ 63 private static final int ACQUIRE_SUCCESS = 0; 64 /** 65 * Returned by nativeImageSetup when we couldn't acquire the buffer, 66 * because there were no buffers available to acquire. 67 */ 68 private static final int ACQUIRE_NO_BUFS = 1; 69 /** 70 * Returned by nativeImageSetup when we couldn't acquire the buffer 71 * because the consumer has already acquired {@maxImages} and cannot 72 * acquire more than that. 73 */ 74 private static final int ACQUIRE_MAX_IMAGES = 2; 75 76 /** 77 * Invalid consumer buffer usage flag. This usage flag will be ignored 78 * by the {@code ImageReader} instance is constructed with this value. 79 */ 80 private static final long BUFFER_USAGE_UNKNOWN = 0; 81 82 /** 83 * <p> 84 * Create a new reader for images of the desired size and format. 85 * </p> 86 * <p> 87 * The {@code maxImages} parameter determines the maximum number of 88 * {@link Image} objects that can be be acquired from the 89 * {@code ImageReader} simultaneously. Requesting more buffers will use up 90 * more memory, so it is important to use only the minimum number necessary 91 * for the use case. 92 * </p> 93 * <p> 94 * The valid sizes and formats depend on the source of the image data. 95 * </p> 96 * <p> 97 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created 98 * {@link ImageReader} will produce images that are not directly accessible 99 * by the application. The application can still acquire images from this 100 * {@link ImageReader}, and send them to the 101 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via 102 * {@link ImageWriter} interface. However, the {@link Image#getPlanes() 103 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE 104 * PRIVATE} format images. The application can check if an existing reader's 105 * format by calling {@link #getImageFormat()}. 106 * </p> 107 * <p> 108 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader 109 * ImageReaders} are more efficient to use when application access to image 110 * data is not necessary, compared to ImageReaders using other format such 111 * as {@link ImageFormat#YUV_420_888 YUV_420_888}. 112 * </p> 113 * 114 * @param width The default width in pixels of the Images that this reader 115 * will produce. 116 * @param height The default height in pixels of the Images that this reader 117 * will produce. 118 * @param format The format of the Image that this reader will produce. This 119 * must be one of the {@link android.graphics.ImageFormat} or 120 * {@link android.graphics.PixelFormat} constants. Note that not 121 * all formats are supported, like ImageFormat.NV21. 122 * @param maxImages The maximum number of images the user will want to 123 * access simultaneously. This should be as small as possible to 124 * limit memory use. Once maxImages Images are obtained by the 125 * user, one of them has to be released before a new Image will 126 * become available for access through 127 * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 128 * Must be greater than 0. 129 * @see Image 130 */ newInstance(int width, int height, int format, int maxImages)131 public static ImageReader newInstance(int width, int height, int format, int maxImages) { 132 return new ImageReader(width, height, format, maxImages, BUFFER_USAGE_UNKNOWN); 133 } 134 135 /** 136 * <p> 137 * Create a new reader for images of the desired size, format and consumer usage flag. 138 * </p> 139 * <p> 140 * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that 141 * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will 142 * use up more memory, so it is important to use only the minimum number necessary for the use 143 * case. 144 * </p> 145 * <p> 146 * The valid sizes and formats depend on the source of the image data. 147 * </p> 148 * <p> 149 * The format and usage flag combination describes how the buffer will be used by 150 * consumer end-points. For example, if the application intends to send the images to 151 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video 152 * encoding, the format and usage flag combination needs to be 153 * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE0_VIDEO_ENCODE}. When an 154 * {@link ImageReader} object is created with a valid size and such format/usage flag 155 * combination, the application can send the {@link Image images} to an {@link ImageWriter} that 156 * is created with the input {@link android.view.Surface} provided by the 157 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. 158 * </p> 159 * <p> 160 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} 161 * will produce images that are not directly accessible by the application. The application can 162 * still acquire images from this {@link ImageReader}, and send them to the 163 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the 164 * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video 165 * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() 166 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format 167 * images. The application can check if an existing reader's format by calling 168 * {@link #getImageFormat()}. 169 * </p> 170 * <p> 171 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more 172 * efficient to use when application access to image data is not necessary, compared to 173 * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. 174 * </p> 175 * <p> 176 * Note that not all format and usage flag combination is supported by the 177 * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} 178 * (assuming the consumer end-points support the such image consumption, e.g., hardware video 179 * encoding). 180 * <table> 181 * <tr> 182 * <th>Format</th> 183 * <th>Compatible usage flags</th> 184 * </tr> 185 * <tr> 186 * <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by 187 * {@link android.graphics.ImageFormat ImageFormat} or 188 * {@link android.graphics.PixelFormat PixelFormat}</td> 189 * <td>{@link HardwareBuffer#USAGE0_CPU_READ} or 190 * {@link HardwareBuffer#USAGE0_CPU_READ_OFTEN}</td> 191 * </tr> 192 * <tr> 193 * <td>{@link android.graphics.ImageFormat#PRIVATE}</td> 194 * <td>{@link HardwareBuffer#USAGE0_VIDEO_ENCODE} or 195 * {@link HardwareBuffer#USAGE0_GPU_SAMPLED_IMAGE}, or combined</td> 196 * </tr> 197 * </table> 198 * Using other combinations may result in {@link IllegalArgumentException}. 199 * </p> 200 * @param width The default width in pixels of the Images that this reader will produce. 201 * @param height The default height in pixels of the Images that this reader will produce. 202 * @param format The format of the Image that this reader will produce. This must be one of the 203 * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} 204 * constants. Note that not all formats are supported, like ImageFormat.NV21. 205 * @param maxImages The maximum number of images the user will want to access simultaneously. 206 * This should be as small as possible to limit memory use. Once maxImages Images are 207 * obtained by the user, one of them has to be released before a new Image will 208 * become available for access through {@link #acquireLatestImage()} or 209 * {@link #acquireNextImage()}. Must be greater than 0. 210 * @param usage The intended usage of the images produced by this ImageReader. It needs 211 * to be one of the Usage0 defined by {@link HardwareBuffer}, or an 212 * {@link IllegalArgumentException} will be thrown. 213 * @see Image 214 * @see HardwareBuffer 215 * @hide 216 */ newInstance(int width, int height, int format, int maxImages, long usage)217 public static ImageReader newInstance(int width, int height, int format, int maxImages, 218 long usage) { 219 if (!isFormatUsageCombinationAllowed(format, usage)) { 220 throw new IllegalArgumentException("Format usage combination is not supported:" 221 + " format = " + format + ", usage = " + usage); 222 } 223 return new ImageReader(width, height, format, maxImages, usage); 224 } 225 226 /** 227 * @hide 228 */ ImageReader(int width, int height, int format, int maxImages, long usage)229 protected ImageReader(int width, int height, int format, int maxImages, long usage) { 230 mWidth = width; 231 mHeight = height; 232 mFormat = format; 233 mMaxImages = maxImages; 234 235 if (width < 1 || height < 1) { 236 throw new IllegalArgumentException( 237 "The image dimensions must be positive"); 238 } 239 if (mMaxImages < 1) { 240 throw new IllegalArgumentException( 241 "Maximum outstanding image count must be at least 1"); 242 } 243 244 if (format == ImageFormat.NV21) { 245 throw new IllegalArgumentException( 246 "NV21 format is not supported"); 247 } 248 249 mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); 250 251 nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage); 252 253 mSurface = nativeGetSurface(); 254 255 mIsReaderValid = true; 256 // Estimate the native buffer allocation size and register it so it gets accounted for 257 // during GC. Note that this doesn't include the buffers required by the buffer queue 258 // itself and the buffers requested by the producer. 259 // Only include memory for 1 buffer, since actually accounting for the memory used is 260 // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some 261 // size. 262 mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( 263 width, height, format, /*buffer count*/ 1); 264 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 265 } 266 267 /** 268 * The default width of {@link Image Images}, in pixels. 269 * 270 * <p>The width may be overridden by the producer sending buffers to this 271 * ImageReader's Surface. If so, the actual width of the images can be 272 * found using {@link Image#getWidth}.</p> 273 * 274 * @return the expected width of an Image 275 */ getWidth()276 public int getWidth() { 277 return mWidth; 278 } 279 280 /** 281 * The default height of {@link Image Images}, in pixels. 282 * 283 * <p>The height may be overridden by the producer sending buffers to this 284 * ImageReader's Surface. If so, the actual height of the images can be 285 * found using {@link Image#getHeight}.</p> 286 * 287 * @return the expected height of an Image 288 */ getHeight()289 public int getHeight() { 290 return mHeight; 291 } 292 293 /** 294 * The default {@link ImageFormat image format} of {@link Image Images}. 295 * 296 * <p>Some color formats may be overridden by the producer sending buffers to 297 * this ImageReader's Surface if the default color format allows. ImageReader 298 * guarantees that all {@link Image Images} acquired from ImageReader 299 * (for example, with {@link #acquireNextImage}) will have a "compatible" 300 * format to what was specified in {@link #newInstance}. 301 * As of now, each format is only compatible to itself. 302 * The actual format of the images can be found using {@link Image#getFormat}.</p> 303 * 304 * @return the expected format of an Image 305 * 306 * @see ImageFormat 307 */ getImageFormat()308 public int getImageFormat() { 309 return mFormat; 310 } 311 312 /** 313 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 314 * with {@link #acquireNextImage}). 315 * 316 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 317 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 318 * </p> 319 * 320 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 321 * acquire function throwing a {@link IllegalStateException}. Furthermore, 322 * while the max number of images have been acquired by the ImageReader user, the producer 323 * enqueueing additional images may stall until at least one image has been released. </p> 324 * 325 * @return Maximum number of images for this ImageReader. 326 * 327 * @see Image#close 328 */ getMaxImages()329 public int getMaxImages() { 330 return mMaxImages; 331 } 332 333 /** 334 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 335 * {@code ImageReader}.</p> 336 * 337 * <p>Until valid image data is rendered into this {@link Surface}, the 338 * {@link #acquireNextImage} method will return {@code null}. Only one source 339 * can be producing data into this Surface at the same time, although the 340 * same {@link Surface} can be reused with a different API once the first source is 341 * disconnected from the {@link Surface}.</p> 342 * 343 * <p>Please note that holding on to the Surface object returned by this method is not enough 344 * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a 345 * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.</p> 346 * 347 * @return A {@link Surface} to use for a drawing target for various APIs. 348 */ getSurface()349 public Surface getSurface() { 350 return mSurface; 351 } 352 353 /** 354 * <p> 355 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 356 * {@link Image images}. Returns {@code null} if no new image is available. 357 * </p> 358 * <p> 359 * This operation will acquire all the images possible from the ImageReader, 360 * but {@link #close} all images that aren't the latest. This function is 361 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 362 * more suited for real-time processing. 363 * </p> 364 * <p> 365 * Note that {@link #getMaxImages maxImages} should be at least 2 for 366 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 367 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 368 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 369 * with less than two images of margin, that is 370 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 371 * </p> 372 * <p> 373 * This operation will fail by throwing an {@link IllegalStateException} if 374 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 375 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 376 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 377 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 378 * will be thrown until more images are 379 * released with {@link Image#close}. 380 * </p> 381 * 382 * @return latest frame of image data, or {@code null} if no image data is available. 383 * @throws IllegalStateException if too many images are currently acquired 384 */ acquireLatestImage()385 public Image acquireLatestImage() { 386 Image image = acquireNextImage(); 387 if (image == null) { 388 return null; 389 } 390 try { 391 for (;;) { 392 Image next = acquireNextImageNoThrowISE(); 393 if (next == null) { 394 Image result = image; 395 image = null; 396 return result; 397 } 398 image.close(); 399 image = next; 400 } 401 } finally { 402 if (image != null) { 403 image.close(); 404 } 405 } 406 } 407 408 /** 409 * Don't throw IllegalStateException if there are too many images acquired. 410 * 411 * @return Image if acquiring succeeded, or null otherwise. 412 * 413 * @hide 414 */ acquireNextImageNoThrowISE()415 public Image acquireNextImageNoThrowISE() { 416 SurfaceImage si = new SurfaceImage(mFormat); 417 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 418 } 419 420 /** 421 * Attempts to acquire the next image from the underlying native implementation. 422 * 423 * <p> 424 * Note that unexpected failures will throw at the JNI level. 425 * </p> 426 * 427 * @param si A blank SurfaceImage. 428 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 429 * 430 * @see #ACQUIRE_MAX_IMAGES 431 * @see #ACQUIRE_NO_BUFS 432 * @see #ACQUIRE_SUCCESS 433 */ acquireNextSurfaceImage(SurfaceImage si)434 private int acquireNextSurfaceImage(SurfaceImage si) { 435 synchronized (mCloseLock) { 436 // A null image will eventually be returned if ImageReader is already closed. 437 int status = ACQUIRE_NO_BUFS; 438 if (mIsReaderValid) { 439 status = nativeImageSetup(si); 440 } 441 442 switch (status) { 443 case ACQUIRE_SUCCESS: 444 si.mIsImageValid = true; 445 case ACQUIRE_NO_BUFS: 446 case ACQUIRE_MAX_IMAGES: 447 break; 448 default: 449 throw new AssertionError("Unknown nativeImageSetup return code " + status); 450 } 451 452 // Only keep track the successfully acquired image, as the native buffer is only mapped 453 // for such case. 454 if (status == ACQUIRE_SUCCESS) { 455 mAcquiredImages.add(si); 456 } 457 return status; 458 } 459 } 460 461 /** 462 * <p> 463 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 464 * no new image is available. 465 * </p> 466 * 467 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 468 * automatically release older images, and allow slower-running processing routines to catch 469 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 470 * batch/background processing. Incorrectly using this function can cause images to appear 471 * with an ever-increasing delay, followed by a complete stall where no new images seem to 472 * appear. 473 * </p> 474 * 475 * <p> 476 * This operation will fail by throwing an {@link IllegalStateException} if 477 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 478 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 479 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 480 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 481 * {@link IllegalStateException} will be thrown until more images are released with 482 * {@link Image#close}. 483 * </p> 484 * 485 * @return a new frame of image data, or {@code null} if no image data is available. 486 * @throws IllegalStateException if {@code maxImages} images are currently acquired 487 * @see #acquireLatestImage 488 */ acquireNextImage()489 public Image acquireNextImage() { 490 // Initialize with reader format, but can be overwritten by native if the image 491 // format is different from the reader format. 492 SurfaceImage si = new SurfaceImage(mFormat); 493 int status = acquireNextSurfaceImage(si); 494 495 switch (status) { 496 case ACQUIRE_SUCCESS: 497 return si; 498 case ACQUIRE_NO_BUFS: 499 return null; 500 case ACQUIRE_MAX_IMAGES: 501 throw new IllegalStateException( 502 String.format( 503 "maxImages (%d) has already been acquired, " + 504 "call #close before acquiring more.", mMaxImages)); 505 default: 506 throw new AssertionError("Unknown nativeImageSetup return code " + status); 507 } 508 } 509 510 /** 511 * <p>Return the frame to the ImageReader for reuse.</p> 512 */ releaseImage(Image i)513 private void releaseImage(Image i) { 514 if (! (i instanceof SurfaceImage) ) { 515 throw new IllegalArgumentException( 516 "This image was not produced by an ImageReader"); 517 } 518 SurfaceImage si = (SurfaceImage) i; 519 if (si.mIsImageValid == false) { 520 return; 521 } 522 523 if (si.getReader() != this || !mAcquiredImages.contains(i)) { 524 throw new IllegalArgumentException( 525 "This image was not produced by this ImageReader"); 526 } 527 528 si.clearSurfacePlanes(); 529 nativeReleaseImage(i); 530 si.mIsImageValid = false; 531 mAcquiredImages.remove(i); 532 } 533 534 /** 535 * Register a listener to be invoked when a new image becomes available 536 * from the ImageReader. 537 * 538 * @param listener 539 * The listener that will be run. 540 * @param handler 541 * The handler on which the listener should be invoked, or null 542 * if the listener should be invoked on the calling thread's looper. 543 * @throws IllegalArgumentException 544 * If no handler specified and the calling thread has no looper. 545 */ setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler)546 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 547 synchronized (mListenerLock) { 548 if (listener != null) { 549 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 550 if (looper == null) { 551 throw new IllegalArgumentException( 552 "handler is null but the current thread is not a looper"); 553 } 554 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 555 mListenerHandler = new ListenerHandler(looper); 556 } 557 mListener = listener; 558 } else { 559 mListener = null; 560 mListenerHandler = null; 561 } 562 } 563 } 564 565 /** 566 * Callback interface for being notified that a new image is available. 567 * 568 * <p> 569 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 570 * available from ImageReader. 571 * </p> 572 */ 573 public interface OnImageAvailableListener { 574 /** 575 * Callback that is called when a new image is available from ImageReader. 576 * 577 * @param reader the ImageReader the callback is associated with. 578 * @see ImageReader 579 * @see Image 580 */ onImageAvailable(ImageReader reader)581 void onImageAvailable(ImageReader reader); 582 } 583 584 /** 585 * Free up all the resources associated with this ImageReader. 586 * 587 * <p> 588 * After calling this method, this ImageReader can not be used. Calling 589 * any methods on this ImageReader and Images previously provided by 590 * {@link #acquireNextImage} or {@link #acquireLatestImage} 591 * will result in an {@link IllegalStateException}, and attempting to read from 592 * {@link ByteBuffer ByteBuffers} returned by an earlier 593 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 594 * have undefined behavior. 595 * </p> 596 */ 597 @Override close()598 public void close() { 599 setOnImageAvailableListener(null, null); 600 if (mSurface != null) mSurface.release(); 601 602 /** 603 * Close all outstanding acquired images before closing the ImageReader. It is a good 604 * practice to close all the images as soon as it is not used to reduce system instantaneous 605 * memory pressure. CopyOnWrite list will use a copy of current list content. For the images 606 * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For 607 * the image being acquired by other threads, mCloseLock is used to synchronize close and 608 * acquire operations. 609 */ 610 synchronized (mCloseLock) { 611 mIsReaderValid = false; 612 for (Image image : mAcquiredImages) { 613 image.close(); 614 } 615 mAcquiredImages.clear(); 616 617 nativeClose(); 618 619 if (mEstimatedNativeAllocBytes > 0) { 620 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 621 mEstimatedNativeAllocBytes = 0; 622 } 623 } 624 } 625 626 /** 627 * Discard any free buffers owned by this ImageReader. 628 * 629 * <p> 630 * Generally, the ImageReader caches buffers for reuse once they have been 631 * allocated, for best performance. However, sometimes it may be important to 632 * release all the cached, unused buffers to save on memory. 633 * </p> 634 * <p> 635 * Calling this method will discard all free cached buffers. This does not include any buffers 636 * associated with Images acquired from the ImageReader, any filled buffers waiting to be 637 * acquired, and any buffers currently in use by the source rendering buffers into the 638 * ImageReader's Surface. 639 * <p> 640 * The ImageReader continues to be usable after this call, but may need to reallocate buffers 641 * when more buffers are needed for rendering. 642 * </p> 643 * @hide 644 */ discardFreeBuffers()645 public void discardFreeBuffers() { 646 synchronized (mCloseLock) { 647 nativeDiscardFreeBuffers(); 648 } 649 } 650 651 @Override finalize()652 protected void finalize() throws Throwable { 653 try { 654 close(); 655 } finally { 656 super.finalize(); 657 } 658 } 659 660 /** 661 * <p> 662 * Remove the ownership of this image from the ImageReader. 663 * </p> 664 * <p> 665 * After this call, the ImageReader no longer owns this image, and the image 666 * ownership can be transfered to another entity like {@link ImageWriter} 667 * via {@link ImageWriter#queueInputImage}. It's up to the new owner to 668 * release the resources held by this image. For example, if the ownership 669 * of this image is transfered to an {@link ImageWriter}, the image will be 670 * freed by the ImageWriter after the image data consumption is done. 671 * </p> 672 * <p> 673 * This method can be used to achieve zero buffer copy for use cases like 674 * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV 675 * reprocessing, where the application can select an output image from 676 * {@link ImageReader} and transfer this image directly to 677 * {@link ImageWriter}, where this image can be consumed by camera directly. 678 * For PRIVATE reprocessing, this is the only way to send input buffers to 679 * the {@link android.hardware.camera2.CameraDevice camera} for 680 * reprocessing. 681 * </p> 682 * <p> 683 * This is a package private method that is only used internally. 684 * </p> 685 * 686 * @param image The image to be detached from this ImageReader. 687 * @throws IllegalStateException If the ImageReader or image have been 688 * closed, or the has been detached, or has not yet been 689 * acquired. 690 */ detachImage(Image image)691 void detachImage(Image image) { 692 if (image == null) { 693 throw new IllegalArgumentException("input image must not be null"); 694 } 695 if (!isImageOwnedbyMe(image)) { 696 throw new IllegalArgumentException("Trying to detach an image that is not owned by" 697 + " this ImageReader"); 698 } 699 700 SurfaceImage si = (SurfaceImage) image; 701 si.throwISEIfImageIsInvalid(); 702 703 if (si.isAttachable()) { 704 throw new IllegalStateException("Image was already detached from this ImageReader"); 705 } 706 707 nativeDetachImage(image); 708 si.clearSurfacePlanes(); 709 si.mPlanes = null; 710 si.setDetached(true); 711 } 712 isImageOwnedbyMe(Image image)713 private boolean isImageOwnedbyMe(Image image) { 714 if (!(image instanceof SurfaceImage)) { 715 return false; 716 } 717 SurfaceImage si = (SurfaceImage) image; 718 return si.getReader() == this; 719 } 720 isFormatUsageCombinationAllowed(int format, long usage)721 private static boolean isFormatUsageCombinationAllowed(int format, long usage) { 722 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 723 return false; 724 } 725 726 // Valid usage needs to be provided. 727 if (usage == BUFFER_USAGE_UNKNOWN) { 728 return false; 729 } 730 731 if (format == ImageFormat.PRIVATE) { 732 // Usage need to be either USAGE0_GPU_SAMPLED_IMAGE or USAGE0_VIDEO_ENCODE or combined. 733 boolean isAllowed = (usage == HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE); 734 isAllowed = isAllowed || (usage == HardwareBuffer.USAGE_VIDEO_ENCODE); 735 isAllowed = isAllowed || (usage == 736 (HardwareBuffer.USAGE_VIDEO_ENCODE | HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE)); 737 return isAllowed; 738 } else { 739 // Usage need to make the buffer CPU readable for explicit format. 740 return ((usage == HardwareBuffer.USAGE_CPU_READ_RARELY) || 741 (usage == HardwareBuffer.USAGE_CPU_READ_OFTEN)); 742 } 743 } 744 745 /** 746 * Called from Native code when an Event happens. 747 * 748 * This may be called from an arbitrary Binder thread, so access to the ImageReader must be 749 * synchronized appropriately. 750 */ postEventFromNative(Object selfRef)751 private static void postEventFromNative(Object selfRef) { 752 @SuppressWarnings("unchecked") 753 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 754 final ImageReader ir = weakSelf.get(); 755 if (ir == null) { 756 return; 757 } 758 759 final Handler handler; 760 synchronized (ir.mListenerLock) { 761 handler = ir.mListenerHandler; 762 } 763 if (handler != null) { 764 handler.sendEmptyMessage(0); 765 } 766 } 767 768 private final int mWidth; 769 private final int mHeight; 770 private final int mFormat; 771 private final int mMaxImages; 772 private final int mNumPlanes; 773 private final Surface mSurface; 774 private int mEstimatedNativeAllocBytes; 775 776 private final Object mListenerLock = new Object(); 777 private final Object mCloseLock = new Object(); 778 private boolean mIsReaderValid = false; 779 private OnImageAvailableListener mListener; 780 private ListenerHandler mListenerHandler; 781 // Keep track of the successfully acquired Images. This need to be thread safe as the images 782 // could be closed by different threads (e.g., application thread and GC thread). 783 private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>(); 784 785 /** 786 * This field is used by native code, do not access or modify. 787 */ 788 private long mNativeContext; 789 790 /** 791 * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. 792 */ 793 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)794 public ListenerHandler(Looper looper) { 795 super(looper, null, true /*async*/); 796 } 797 798 @Override handleMessage(Message msg)799 public void handleMessage(Message msg) { 800 OnImageAvailableListener listener; 801 synchronized (mListenerLock) { 802 listener = mListener; 803 } 804 805 // It's dangerous to fire onImageAvailable() callback when the ImageReader is being 806 // closed, as application could acquire next image in the onImageAvailable() callback. 807 boolean isReaderValid = false; 808 synchronized (mCloseLock) { 809 isReaderValid = mIsReaderValid; 810 } 811 if (listener != null && isReaderValid) { 812 listener.onImageAvailable(ImageReader.this); 813 } 814 } 815 } 816 817 private class SurfaceImage extends android.media.Image { SurfaceImage(int format)818 public SurfaceImage(int format) { 819 mFormat = format; 820 } 821 822 @Override close()823 public void close() { 824 ImageReader.this.releaseImage(this); 825 } 826 getReader()827 public ImageReader getReader() { 828 return ImageReader.this; 829 } 830 831 @Override getFormat()832 public int getFormat() { 833 throwISEIfImageIsInvalid(); 834 int readerFormat = ImageReader.this.getImageFormat(); 835 // Assume opaque reader always produce opaque images. 836 mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : 837 nativeGetFormat(readerFormat); 838 return mFormat; 839 } 840 841 @Override getWidth()842 public int getWidth() { 843 throwISEIfImageIsInvalid(); 844 int width; 845 switch(getFormat()) { 846 case ImageFormat.JPEG: 847 case ImageFormat.DEPTH_POINT_CLOUD: 848 case ImageFormat.RAW_PRIVATE: 849 width = ImageReader.this.getWidth(); 850 break; 851 default: 852 width = nativeGetWidth(); 853 } 854 return width; 855 } 856 857 @Override getHeight()858 public int getHeight() { 859 throwISEIfImageIsInvalid(); 860 int height; 861 switch(getFormat()) { 862 case ImageFormat.JPEG: 863 case ImageFormat.DEPTH_POINT_CLOUD: 864 case ImageFormat.RAW_PRIVATE: 865 height = ImageReader.this.getHeight(); 866 break; 867 default: 868 height = nativeGetHeight(); 869 } 870 return height; 871 } 872 873 @Override getTimestamp()874 public long getTimestamp() { 875 throwISEIfImageIsInvalid(); 876 return mTimestamp; 877 } 878 879 @Override setTimestamp(long timestampNs)880 public void setTimestamp(long timestampNs) { 881 throwISEIfImageIsInvalid(); 882 mTimestamp = timestampNs; 883 } 884 885 @Override getPlanes()886 public Plane[] getPlanes() { 887 throwISEIfImageIsInvalid(); 888 889 if (mPlanes == null) { 890 mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat); 891 } 892 // Shallow copy is fine. 893 return mPlanes.clone(); 894 } 895 896 @Override finalize()897 protected final void finalize() throws Throwable { 898 try { 899 close(); 900 } finally { 901 super.finalize(); 902 } 903 } 904 905 @Override isAttachable()906 boolean isAttachable() { 907 throwISEIfImageIsInvalid(); 908 return mIsDetached.get(); 909 } 910 911 @Override getOwner()912 ImageReader getOwner() { 913 throwISEIfImageIsInvalid(); 914 return ImageReader.this; 915 } 916 917 @Override getNativeContext()918 long getNativeContext() { 919 throwISEIfImageIsInvalid(); 920 return mNativeBuffer; 921 } 922 setDetached(boolean detached)923 private void setDetached(boolean detached) { 924 throwISEIfImageIsInvalid(); 925 mIsDetached.getAndSet(detached); 926 } 927 clearSurfacePlanes()928 private void clearSurfacePlanes() { 929 // Image#getPlanes may not be called before the image is closed. 930 if (mIsImageValid && mPlanes != null) { 931 for (int i = 0; i < mPlanes.length; i++) { 932 if (mPlanes[i] != null) { 933 mPlanes[i].clearBuffer(); 934 mPlanes[i] = null; 935 } 936 } 937 } 938 } 939 940 private class SurfacePlane extends android.media.Image.Plane { 941 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 942 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)943 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 944 mRowStride = rowStride; 945 mPixelStride = pixelStride; 946 mBuffer = buffer; 947 /** 948 * Set the byteBuffer order according to host endianness (native 949 * order), otherwise, the byteBuffer order defaults to 950 * ByteOrder.BIG_ENDIAN. 951 */ 952 mBuffer.order(ByteOrder.nativeOrder()); 953 } 954 955 @Override getBuffer()956 public ByteBuffer getBuffer() { 957 throwISEIfImageIsInvalid(); 958 return mBuffer; 959 } 960 961 @Override getPixelStride()962 public int getPixelStride() { 963 SurfaceImage.this.throwISEIfImageIsInvalid(); 964 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 965 throw new UnsupportedOperationException( 966 "getPixelStride is not supported for RAW_PRIVATE plane"); 967 } 968 return mPixelStride; 969 } 970 971 @Override getRowStride()972 public int getRowStride() { 973 SurfaceImage.this.throwISEIfImageIsInvalid(); 974 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 975 throw new UnsupportedOperationException( 976 "getRowStride is not supported for RAW_PRIVATE plane"); 977 } 978 return mRowStride; 979 } 980 clearBuffer()981 private void clearBuffer() { 982 // Need null check first, as the getBuffer() may not be called before an image 983 // is closed. 984 if (mBuffer == null) { 985 return; 986 } 987 988 if (mBuffer.isDirect()) { 989 NioUtils.freeDirectBuffer(mBuffer); 990 } 991 mBuffer = null; 992 } 993 994 final private int mPixelStride; 995 final private int mRowStride; 996 997 private ByteBuffer mBuffer; 998 } 999 1000 /** 1001 * This field is used to keep track of native object and used by native code only. 1002 * Don't modify. 1003 */ 1004 private long mNativeBuffer; 1005 1006 /** 1007 * This field is set by native code during nativeImageSetup(). 1008 */ 1009 private long mTimestamp; 1010 1011 private SurfacePlane[] mPlanes; 1012 private int mFormat = ImageFormat.UNKNOWN; 1013 // If this image is detached from the ImageReader. 1014 private AtomicBoolean mIsDetached = new AtomicBoolean(false); 1015 nativeCreatePlanes(int numPlanes, int readerFormat)1016 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, 1017 int readerFormat); nativeGetWidth()1018 private synchronized native int nativeGetWidth(); nativeGetHeight()1019 private synchronized native int nativeGetHeight(); nativeGetFormat(int readerFormat)1020 private synchronized native int nativeGetFormat(int readerFormat); 1021 } 1022 nativeInit(Object weakSelf, int w, int h, int fmt, int maxImgs, long consumerUsage)1023 private synchronized native void nativeInit(Object weakSelf, int w, int h, 1024 int fmt, int maxImgs, long consumerUsage); nativeClose()1025 private synchronized native void nativeClose(); nativeReleaseImage(Image i)1026 private synchronized native void nativeReleaseImage(Image i); nativeGetSurface()1027 private synchronized native Surface nativeGetSurface(); nativeDetachImage(Image i)1028 private synchronized native int nativeDetachImage(Image i); nativeDiscardFreeBuffers()1029 private synchronized native void nativeDiscardFreeBuffers(); 1030 1031 /** 1032 * @return A return code {@code ACQUIRE_*} 1033 * 1034 * @see #ACQUIRE_SUCCESS 1035 * @see #ACQUIRE_NO_BUFS 1036 * @see #ACQUIRE_MAX_IMAGES 1037 */ nativeImageSetup(Image i)1038 private synchronized native int nativeImageSetup(Image i); 1039 1040 /** 1041 * We use a class initializer to allow the native code to cache some 1042 * field offsets. 1043 */ nativeClassInit()1044 private static native void nativeClassInit(); 1045 static { 1046 System.loadLibrary("media_jni"); nativeClassInit()1047 nativeClassInit(); 1048 } 1049 } 1050