1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.graphics.ImageFormat; 22 import android.graphics.ImageFormat.Format; 23 import android.graphics.PixelFormat; 24 import android.graphics.Rect; 25 import android.hardware.camera2.utils.SurfaceUtils; 26 import android.hardware.HardwareBuffer; 27 import android.os.Handler; 28 import android.os.Looper; 29 import android.os.Message; 30 import android.util.Size; 31 import android.view.Surface; 32 33 import dalvik.system.VMRuntime; 34 35 import java.lang.ref.WeakReference; 36 import java.nio.ByteBuffer; 37 import java.nio.ByteOrder; 38 import java.nio.NioUtils; 39 import java.util.List; 40 import java.util.concurrent.CopyOnWriteArrayList; 41 42 /** 43 * <p> 44 * The ImageWriter class allows an application to produce Image data into a 45 * {@link android.view.Surface}, and have it be consumed by another component 46 * like {@link android.hardware.camera2.CameraDevice CameraDevice}. 47 * </p> 48 * <p> 49 * Several Android API classes can provide input {@link android.view.Surface 50 * Surface} objects for ImageWriter to produce data into, including 51 * {@link MediaCodec MediaCodec} (encoder), 52 * {@link android.hardware.camera2.CameraCaptureSession CameraCaptureSession} 53 * (reprocessing input), {@link ImageReader}, etc. 54 * </p> 55 * <p> 56 * The input Image data is encapsulated in {@link Image} objects. To produce 57 * Image data into a destination {@link android.view.Surface Surface}, the 58 * application can get an input Image via {@link #dequeueInputImage} then write 59 * Image data into it. Multiple such {@link Image} objects can be dequeued at 60 * the same time and queued back in any order, up to the number specified by the 61 * {@code maxImages} constructor parameter. 62 * </p> 63 * <p> 64 * If the application already has an Image from {@link ImageReader}, the 65 * application can directly queue this Image into the ImageWriter (via 66 * {@link #queueInputImage}), potentially with zero buffer copies. This 67 * even works if the image format of the ImageWriter is 68 * {@link ImageFormat#PRIVATE PRIVATE}, and prior to Android P is the only 69 * way to enqueue images into such an ImageWriter. Starting in Android P 70 * private images may also be accessed through their hardware buffers 71 * (when available) through the {@link Image#getHardwareBuffer()} method. 72 * Attempting to access the planes of a private image, will return an 73 * empty array. 74 * </p> 75 * <p> 76 * Once new input Images are queued into an ImageWriter, it's up to the 77 * downstream components (e.g. {@link ImageReader} or 78 * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the 79 * downstream components cannot consume the Images at least as fast as the 80 * ImageWriter production rate, the {@link #dequeueInputImage} call will 81 * eventually block and the application will have to drop input frames. 82 * </p> 83 * <p> 84 * If the consumer component that provided the input {@link android.view.Surface Surface} 85 * abandons the {@link android.view.Surface Surface}, {@link #queueInputImage queueing} 86 * or {@link #dequeueInputImage dequeueing} an {@link Image} will throw an 87 * {@link IllegalStateException}. 88 * </p> 89 */ 90 public class ImageWriter implements AutoCloseable { 91 private final Object mListenerLock = new Object(); 92 private OnImageReleasedListener mListener; 93 private ListenerHandler mListenerHandler; 94 private long mNativeContext; 95 96 // Field below is used by native code, do not access or modify. 97 private int mWriterFormat; 98 99 private final int mMaxImages; 100 // Keep track of the currently dequeued Image. This need to be thread safe as the images 101 // could be closed by different threads (e.g., application thread and GC thread). 102 private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>(); 103 private int mEstimatedNativeAllocBytes; 104 105 /** 106 * <p> 107 * Create a new ImageWriter. 108 * </p> 109 * <p> 110 * The {@code maxImages} parameter determines the maximum number of 111 * {@link Image} objects that can be be dequeued from the 112 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 113 * more memory, so it is important to use only the minimum number necessary. 114 * </p> 115 * <p> 116 * The input Image size and format depend on the Surface that is provided by 117 * the downstream consumer end-point. 118 * </p> 119 * 120 * @param surface The destination Surface this writer produces Image data 121 * into. 122 * @param maxImages The maximum number of Images the user will want to 123 * access simultaneously for producing Image data. This should be 124 * as small as possible to limit memory use. Once maxImages 125 * Images are dequeued by the user, one of them has to be queued 126 * back before a new Image can be dequeued for access via 127 * {@link #dequeueInputImage()}. 128 * @return a new ImageWriter instance. 129 */ newInstance(@onNull Surface surface, @IntRange(from = 1) int maxImages)130 public static @NonNull ImageWriter newInstance(@NonNull Surface surface, 131 @IntRange(from = 1) int maxImages) { 132 return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN); 133 } 134 135 /** 136 * <p> 137 * Create a new ImageWriter with given number of max Images and format. 138 * </p> 139 * <p> 140 * The {@code maxImages} parameter determines the maximum number of 141 * {@link Image} objects that can be be dequeued from the 142 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 143 * more memory, so it is important to use only the minimum number necessary. 144 * </p> 145 * <p> 146 * The format specifies the image format of this ImageWriter. The format 147 * from the {@code surface} will be overridden with this format. For example, 148 * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default 149 * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter 150 * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate 151 * with {@link ImageFormat#PRIVATE} Images. 152 * </p> 153 * <p> 154 * Note that the consumer end-point may or may not be able to support Images with different 155 * format, for such case, the application should only use this method if the consumer is able 156 * to consume such images. 157 * </p> 158 * <p> 159 * The input Image size depends on the Surface that is provided by 160 * the downstream consumer end-point. 161 * </p> 162 * 163 * @param surface The destination Surface this writer produces Image data 164 * into. 165 * @param maxImages The maximum number of Images the user will want to 166 * access simultaneously for producing Image data. This should be 167 * as small as possible to limit memory use. Once maxImages 168 * Images are dequeued by the user, one of them has to be queued 169 * back before a new Image can be dequeued for access via 170 * {@link #dequeueInputImage()}. 171 * @param format The format of this ImageWriter. It can be any valid format specified by 172 * {@link ImageFormat} or {@link PixelFormat}. 173 * 174 * @return a new ImageWriter instance. 175 */ newInstance(@onNull Surface surface, @IntRange(from = 1) int maxImages, @Format int format)176 public static @NonNull ImageWriter newInstance(@NonNull Surface surface, 177 @IntRange(from = 1) int maxImages, @Format int format) { 178 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 179 throw new IllegalArgumentException("Invalid format is specified: " + format); 180 } 181 return new ImageWriter(surface, maxImages, format); 182 } 183 184 /** 185 * @hide 186 */ ImageWriter(Surface surface, int maxImages, int format)187 protected ImageWriter(Surface surface, int maxImages, int format) { 188 if (surface == null || maxImages < 1) { 189 throw new IllegalArgumentException("Illegal input argument: surface " + surface 190 + ", maxImages: " + maxImages); 191 } 192 193 mMaxImages = maxImages; 194 195 if (format == ImageFormat.UNKNOWN) { 196 format = SurfaceUtils.getSurfaceFormat(surface); 197 } 198 // Note that the underlying BufferQueue is working in synchronous mode 199 // to avoid dropping any buffers. 200 mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format); 201 202 // Estimate the native buffer allocation size and register it so it gets accounted for 203 // during GC. Note that this doesn't include the buffers required by the buffer queue 204 // itself and the buffers requested by the producer. 205 // Only include memory for 1 buffer, since actually accounting for the memory used is 206 // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some 207 // size. 208 Size surfSize = SurfaceUtils.getSurfaceSize(surface); 209 mEstimatedNativeAllocBytes = 210 ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(), 211 format, /*buffer count*/ 1); 212 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 213 } 214 215 /** 216 * <p> 217 * Maximum number of Images that can be dequeued from the ImageWriter 218 * simultaneously (for example, with {@link #dequeueInputImage()}). 219 * </p> 220 * <p> 221 * An Image is considered dequeued after it's returned by 222 * {@link #dequeueInputImage()} from ImageWriter, and until the Image is 223 * sent back to ImageWriter via {@link #queueInputImage}, or 224 * {@link Image#close()}. 225 * </p> 226 * <p> 227 * Attempting to dequeue more than {@code maxImages} concurrently will 228 * result in the {@link #dequeueInputImage()} function throwing an 229 * {@link IllegalStateException}. 230 * </p> 231 * 232 * @return Maximum number of Images that can be dequeued from this 233 * ImageWriter. 234 * @see #dequeueInputImage 235 * @see #queueInputImage 236 * @see Image#close 237 */ getMaxImages()238 public int getMaxImages() { 239 return mMaxImages; 240 } 241 242 /** 243 * <p> 244 * Dequeue the next available input Image for the application to produce 245 * data into. 246 * </p> 247 * <p> 248 * This method requests a new input Image from ImageWriter. The application 249 * owns this Image after this call. Once the application fills the Image 250 * data, it is expected to return this Image back to ImageWriter for 251 * downstream consumer components (e.g. 252 * {@link android.hardware.camera2.CameraDevice}) to consume. The Image can 253 * be returned to ImageWriter via {@link #queueInputImage} or 254 * {@link Image#close()}. 255 * </p> 256 * <p> 257 * This call will block if all available input images have been queued by 258 * the application and the downstream consumer has not yet consumed any. 259 * When an Image is consumed by the downstream consumer and released, an 260 * {@link OnImageReleasedListener#onImageReleased} callback will be fired, 261 * which indicates that there is one input Image available. For non- 262 * {@link ImageFormat#PRIVATE PRIVATE} formats ( 263 * {@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE}), it is 264 * recommended to dequeue the next Image only after this callback is fired, 265 * in the steady state. 266 * </p> 267 * <p> 268 * If the format of ImageWriter is {@link ImageFormat#PRIVATE PRIVATE} ( 269 * {@link ImageWriter#getFormat()} == {@link ImageFormat#PRIVATE}), the 270 * image buffer is accessible to the application only through the hardware 271 * buffer obtained through {@link Image#getHardwareBuffer()}. (On Android 272 * versions prior to P, dequeueing private buffers will cause an 273 * {@link IllegalStateException} to be thrown). Alternatively, 274 * the application can acquire images from some other component (e.g. an 275 * {@link ImageReader}), and queue them directly to this ImageWriter via the 276 * {@link ImageWriter#queueInputImage queueInputImage()} method. 277 * </p> 278 * 279 * @return The next available input Image from this ImageWriter. 280 * @throws IllegalStateException if {@code maxImages} Images are currently 281 * dequeued, or the input {@link android.view.Surface Surface} 282 * has been abandoned by the consumer component that provided 283 * the {@link android.view.Surface Surface}. Prior to Android 284 * P, throws if the ImageWriter format is 285 * {@link ImageFormat#PRIVATE PRIVATE}. 286 * @see #queueInputImage 287 * @see Image#close 288 */ dequeueInputImage()289 public Image dequeueInputImage() { 290 if (mDequeuedImages.size() >= mMaxImages) { 291 throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages); 292 } 293 WriterSurfaceImage newImage = new WriterSurfaceImage(this); 294 nativeDequeueInputImage(mNativeContext, newImage); 295 mDequeuedImages.add(newImage); 296 newImage.mIsImageValid = true; 297 return newImage; 298 } 299 300 /** 301 * <p> 302 * Queue an input {@link Image} back to ImageWriter for the downstream 303 * consumer to access. 304 * </p> 305 * <p> 306 * The input {@link Image} could be from ImageReader (acquired via 307 * {@link ImageReader#acquireNextImage} or 308 * {@link ImageReader#acquireLatestImage}), or from this ImageWriter 309 * (acquired via {@link #dequeueInputImage}). In the former case, the Image 310 * data will be moved to this ImageWriter. Note that the Image properties 311 * (size, format, strides, etc.) must be the same as the properties of the 312 * images dequeued from this ImageWriter, or this method will throw an 313 * {@link IllegalArgumentException}. In the latter case, the application has 314 * filled the input image with data. This method then passes the filled 315 * buffer to the downstream consumer. In both cases, it's up to the caller 316 * to ensure that the Image timestamp (in nanoseconds) is correctly set, as 317 * the downstream component may want to use it to indicate the Image data 318 * capture time. 319 * </p> 320 * <p> 321 * After this method is called and the downstream consumer consumes and 322 * releases the Image, an {@link OnImageReleasedListener#onImageReleased} 323 * callback will fire. The application can use this callback to avoid 324 * sending Images faster than the downstream consumer processing rate in 325 * steady state. 326 * </p> 327 * <p> 328 * Passing in an Image from some other component (e.g. an 329 * {@link ImageReader}) requires a free input Image from this ImageWriter as 330 * the destination. In this case, this call will block, as 331 * {@link #dequeueInputImage} does, if there are no free Images available. 332 * To avoid blocking, the application should ensure that there is at least 333 * one free Image available in this ImageWriter before calling this method. 334 * </p> 335 * <p> 336 * After this call, the input Image is no longer valid for further access, 337 * as if the Image is {@link Image#close closed}. Attempting to access the 338 * {@link ByteBuffer ByteBuffers} returned by an earlier 339 * {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an 340 * {@link IllegalStateException}. 341 * </p> 342 * 343 * @param image The Image to be queued back to ImageWriter for future 344 * consumption. 345 * @throws IllegalStateException if the image was already queued previously, 346 * or the image was aborted previously, or the input 347 * {@link android.view.Surface Surface} has been abandoned by the 348 * consumer component that provided the 349 * {@link android.view.Surface Surface}. 350 * @see #dequeueInputImage() 351 */ queueInputImage(Image image)352 public void queueInputImage(Image image) { 353 if (image == null) { 354 throw new IllegalArgumentException("image shouldn't be null"); 355 } 356 boolean ownedByMe = isImageOwnedByMe(image); 357 if (ownedByMe && !(((WriterSurfaceImage) image).mIsImageValid)) { 358 throw new IllegalStateException("Image from ImageWriter is invalid"); 359 } 360 361 // For images from other components, need to detach first, then attach. 362 if (!ownedByMe) { 363 if (!(image.getOwner() instanceof ImageReader)) { 364 throw new IllegalArgumentException("Only images from ImageReader can be queued to" 365 + " ImageWriter, other image source is not supported yet!"); 366 } 367 368 ImageReader prevOwner = (ImageReader) image.getOwner(); 369 370 prevOwner.detachImage(image); 371 attachAndQueueInputImage(image); 372 // This clears the native reference held by the original owner. 373 // When this Image is detached later by this ImageWriter, the 374 // native memory won't be leaked. 375 image.close(); 376 return; 377 } 378 379 Rect crop = image.getCropRect(); 380 nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top, 381 crop.right, crop.bottom, image.getTransform(), image.getScalingMode()); 382 383 /** 384 * Only remove and cleanup the Images that are owned by this 385 * ImageWriter. Images detached from other owners are only temporarily 386 * owned by this ImageWriter and will be detached immediately after they 387 * are released by downstream consumers, so there is no need to keep 388 * track of them in mDequeuedImages. 389 */ 390 if (ownedByMe) { 391 mDequeuedImages.remove(image); 392 // Do not call close here, as close is essentially cancel image. 393 WriterSurfaceImage wi = (WriterSurfaceImage) image; 394 wi.clearSurfacePlanes(); 395 wi.mIsImageValid = false; 396 } 397 } 398 399 /** 400 * Get the ImageWriter format. 401 * <p> 402 * This format may be different than the Image format returned by 403 * {@link Image#getFormat()}. However, if the ImageWriter format is 404 * {@link ImageFormat#PRIVATE PRIVATE}, calling {@link #dequeueInputImage()} 405 * will result in an {@link IllegalStateException}. 406 * </p> 407 * 408 * @return The ImageWriter format. 409 */ getFormat()410 public int getFormat() { 411 return mWriterFormat; 412 } 413 414 /** 415 * ImageWriter callback interface, used to to asynchronously notify the 416 * application of various ImageWriter events. 417 */ 418 public interface OnImageReleasedListener { 419 /** 420 * <p> 421 * Callback that is called when an input Image is released back to 422 * ImageWriter after the data consumption. 423 * </p> 424 * <p> 425 * The client can use this callback to be notified that an input Image 426 * has been consumed and released by the downstream consumer. More 427 * specifically, this callback will be fired for below cases: 428 * <li>The application dequeues an input Image via the 429 * {@link ImageWriter#dequeueInputImage dequeueInputImage()} method, 430 * uses it, and then queues it back to this ImageWriter via the 431 * {@link ImageWriter#queueInputImage queueInputImage()} method. After 432 * the downstream consumer uses and releases this image to this 433 * ImageWriter, this callback will be fired. This image will be 434 * available to be dequeued after this callback.</li> 435 * <li>The application obtains an Image from some other component (e.g. 436 * an {@link ImageReader}), uses it, and then queues it to this 437 * ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}. 438 * After the downstream consumer uses and releases this image to this 439 * ImageWriter, this callback will be fired.</li> 440 * </p> 441 * 442 * @param writer the ImageWriter the callback is associated with. 443 * @see ImageWriter 444 * @see Image 445 */ onImageReleased(ImageWriter writer)446 void onImageReleased(ImageWriter writer); 447 } 448 449 /** 450 * Register a listener to be invoked when an input Image is returned to the 451 * ImageWriter. 452 * 453 * @param listener The listener that will be run. 454 * @param handler The handler on which the listener should be invoked, or 455 * null if the listener should be invoked on the calling thread's 456 * looper. 457 * @throws IllegalArgumentException If no handler specified and the calling 458 * thread has no looper. 459 */ setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler)460 public void setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler) { 461 synchronized (mListenerLock) { 462 if (listener != null) { 463 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 464 if (looper == null) { 465 throw new IllegalArgumentException( 466 "handler is null but the current thread is not a looper"); 467 } 468 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 469 mListenerHandler = new ListenerHandler(looper); 470 } 471 mListener = listener; 472 } else { 473 mListener = null; 474 mListenerHandler = null; 475 } 476 } 477 } 478 479 /** 480 * Free up all the resources associated with this ImageWriter. 481 * <p> 482 * After calling this method, this ImageWriter cannot be used. Calling any 483 * methods on this ImageWriter and Images previously provided by 484 * {@link #dequeueInputImage()} will result in an 485 * {@link IllegalStateException}, and attempting to write into 486 * {@link ByteBuffer ByteBuffers} returned by an earlier 487 * {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined 488 * behavior. 489 * </p> 490 */ 491 @Override close()492 public void close() { 493 setOnImageReleasedListener(null, null); 494 for (Image image : mDequeuedImages) { 495 image.close(); 496 } 497 mDequeuedImages.clear(); 498 nativeClose(mNativeContext); 499 mNativeContext = 0; 500 501 if (mEstimatedNativeAllocBytes > 0) { 502 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 503 mEstimatedNativeAllocBytes = 0; 504 } 505 } 506 507 @Override finalize()508 protected void finalize() throws Throwable { 509 try { 510 close(); 511 } finally { 512 super.finalize(); 513 } 514 } 515 516 /** 517 * <p> 518 * Attach and queue input Image to this ImageWriter. 519 * </p> 520 * <p> 521 * When the format of an Image is {@link ImageFormat#PRIVATE PRIVATE}, or 522 * the source Image is so large that copying its data is too expensive, this 523 * method can be used to migrate the source Image into ImageWriter without a 524 * data copy, and then queue it to this ImageWriter. The source Image must 525 * be detached from its previous owner already, or this call will throw an 526 * {@link IllegalStateException}. 527 * </p> 528 * <p> 529 * After this call, the ImageWriter takes ownership of this Image. This 530 * ownership will automatically be removed from this writer after the 531 * consumer releases this Image, that is, after 532 * {@link OnImageReleasedListener#onImageReleased}. The caller is responsible for 533 * closing this Image through {@link Image#close()} to free up the resources 534 * held by this Image. 535 * </p> 536 * 537 * @param image The source Image to be attached and queued into this 538 * ImageWriter for downstream consumer to use. 539 * @throws IllegalStateException if the Image is not detached from its 540 * previous owner, or the Image is already attached to this 541 * ImageWriter, or the source Image is invalid. 542 */ attachAndQueueInputImage(Image image)543 private void attachAndQueueInputImage(Image image) { 544 if (image == null) { 545 throw new IllegalArgumentException("image shouldn't be null"); 546 } 547 if (isImageOwnedByMe(image)) { 548 throw new IllegalArgumentException( 549 "Can not attach an image that is owned ImageWriter already"); 550 } 551 /** 552 * Throw ISE if the image is not attachable, which means that it is 553 * either owned by other entity now, or completely non-attachable (some 554 * stand-alone images are not backed by native gralloc buffer, thus not 555 * attachable). 556 */ 557 if (!image.isAttachable()) { 558 throw new IllegalStateException("Image was not detached from last owner, or image " 559 + " is not detachable"); 560 } 561 562 // TODO: what if attach failed, throw RTE or detach a slot then attach? 563 // need do some cleanup to make sure no orphaned 564 // buffer caused leak. 565 Rect crop = image.getCropRect(); 566 nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), 567 image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, 568 image.getTransform(), image.getScalingMode()); 569 } 570 571 /** 572 * This custom handler runs asynchronously so callbacks don't get queued 573 * behind UI messages. 574 */ 575 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)576 public ListenerHandler(Looper looper) { 577 super(looper, null, true /* async */); 578 } 579 580 @Override handleMessage(Message msg)581 public void handleMessage(Message msg) { 582 OnImageReleasedListener listener; 583 synchronized (mListenerLock) { 584 listener = mListener; 585 } 586 if (listener != null) { 587 listener.onImageReleased(ImageWriter.this); 588 } 589 } 590 } 591 592 /** 593 * Called from Native code when an Event happens. This may be called from an 594 * arbitrary Binder thread, so access to the ImageWriter must be 595 * synchronized appropriately. 596 */ postEventFromNative(Object selfRef)597 private static void postEventFromNative(Object selfRef) { 598 @SuppressWarnings("unchecked") 599 WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef; 600 final ImageWriter iw = weakSelf.get(); 601 if (iw == null) { 602 return; 603 } 604 605 final Handler handler; 606 synchronized (iw.mListenerLock) { 607 handler = iw.mListenerHandler; 608 } 609 if (handler != null) { 610 handler.sendEmptyMessage(0); 611 } 612 } 613 614 /** 615 * <p> 616 * Abort the Images that were dequeued from this ImageWriter, and return 617 * them to this writer for reuse. 618 * </p> 619 * <p> 620 * This method is used for the cases where the application dequeued the 621 * Image, may have filled the data, but does not want the downstream 622 * component to consume it. The Image will be returned to this ImageWriter 623 * for reuse after this call, and the ImageWriter will immediately have an 624 * Image available to be dequeued. This aborted Image will be invisible to 625 * the downstream consumer, as if nothing happened. 626 * </p> 627 * 628 * @param image The Image to be aborted. 629 * @see #dequeueInputImage() 630 * @see Image#close() 631 */ abortImage(Image image)632 private void abortImage(Image image) { 633 if (image == null) { 634 throw new IllegalArgumentException("image shouldn't be null"); 635 } 636 637 if (!mDequeuedImages.contains(image)) { 638 throw new IllegalStateException("It is illegal to abort some image that is not" 639 + " dequeued yet"); 640 } 641 642 WriterSurfaceImage wi = (WriterSurfaceImage) image; 643 if (!wi.mIsImageValid) { 644 return; 645 } 646 647 /** 648 * We only need abort Images that are owned and dequeued by ImageWriter. 649 * For attached Images, no need to abort, as there are only two cases: 650 * attached + queued successfully, and attach failed. Neither of the 651 * cases need abort. 652 */ 653 cancelImage(mNativeContext, image); 654 mDequeuedImages.remove(image); 655 wi.clearSurfacePlanes(); 656 wi.mIsImageValid = false; 657 } 658 isImageOwnedByMe(Image image)659 private boolean isImageOwnedByMe(Image image) { 660 if (!(image instanceof WriterSurfaceImage)) { 661 return false; 662 } 663 WriterSurfaceImage wi = (WriterSurfaceImage) image; 664 if (wi.getOwner() != this) { 665 return false; 666 } 667 668 return true; 669 } 670 671 private static class WriterSurfaceImage extends android.media.Image { 672 private ImageWriter mOwner; 673 // This field is used by native code, do not access or modify. 674 private long mNativeBuffer; 675 private int mNativeFenceFd = -1; 676 private SurfacePlane[] mPlanes; 677 private int mHeight = -1; 678 private int mWidth = -1; 679 private int mFormat = -1; 680 // When this default timestamp is used, timestamp for the input Image 681 // will be generated automatically when queueInputBuffer is called. 682 private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE; 683 private long mTimestamp = DEFAULT_TIMESTAMP; 684 685 private int mTransform = 0; //Default no transform 686 private int mScalingMode = 0; //Default frozen scaling mode 687 WriterSurfaceImage(ImageWriter writer)688 public WriterSurfaceImage(ImageWriter writer) { 689 mOwner = writer; 690 } 691 692 @Override getFormat()693 public int getFormat() { 694 throwISEIfImageIsInvalid(); 695 696 if (mFormat == -1) { 697 mFormat = nativeGetFormat(); 698 } 699 return mFormat; 700 } 701 702 @Override getWidth()703 public int getWidth() { 704 throwISEIfImageIsInvalid(); 705 706 if (mWidth == -1) { 707 mWidth = nativeGetWidth(); 708 } 709 710 return mWidth; 711 } 712 713 @Override getHeight()714 public int getHeight() { 715 throwISEIfImageIsInvalid(); 716 717 if (mHeight == -1) { 718 mHeight = nativeGetHeight(); 719 } 720 721 return mHeight; 722 } 723 724 @Override getTransform()725 public int getTransform() { 726 throwISEIfImageIsInvalid(); 727 728 return mTransform; 729 } 730 731 @Override getScalingMode()732 public int getScalingMode() { 733 throwISEIfImageIsInvalid(); 734 735 return mScalingMode; 736 } 737 738 @Override getTimestamp()739 public long getTimestamp() { 740 throwISEIfImageIsInvalid(); 741 742 return mTimestamp; 743 } 744 745 @Override setTimestamp(long timestamp)746 public void setTimestamp(long timestamp) { 747 throwISEIfImageIsInvalid(); 748 749 mTimestamp = timestamp; 750 } 751 752 @Override getHardwareBuffer()753 public HardwareBuffer getHardwareBuffer() { 754 throwISEIfImageIsInvalid(); 755 756 return nativeGetHardwareBuffer(); 757 } 758 759 @Override getPlanes()760 public Plane[] getPlanes() { 761 throwISEIfImageIsInvalid(); 762 763 if (mPlanes == null) { 764 int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat()); 765 mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat()); 766 } 767 768 return mPlanes.clone(); 769 } 770 771 @Override isAttachable()772 boolean isAttachable() { 773 throwISEIfImageIsInvalid(); 774 // Don't allow Image to be detached from ImageWriter for now, as no 775 // detach API is exposed. 776 return false; 777 } 778 779 @Override getOwner()780 ImageWriter getOwner() { 781 throwISEIfImageIsInvalid(); 782 783 return mOwner; 784 } 785 786 @Override getNativeContext()787 long getNativeContext() { 788 throwISEIfImageIsInvalid(); 789 790 return mNativeBuffer; 791 } 792 793 @Override close()794 public void close() { 795 if (mIsImageValid) { 796 getOwner().abortImage(this); 797 } 798 } 799 800 @Override finalize()801 protected final void finalize() throws Throwable { 802 try { 803 close(); 804 } finally { 805 super.finalize(); 806 } 807 } 808 clearSurfacePlanes()809 private void clearSurfacePlanes() { 810 if (mIsImageValid && mPlanes != null) { 811 for (int i = 0; i < mPlanes.length; i++) { 812 if (mPlanes[i] != null) { 813 mPlanes[i].clearBuffer(); 814 mPlanes[i] = null; 815 } 816 } 817 } 818 } 819 820 private class SurfacePlane extends android.media.Image.Plane { 821 private ByteBuffer mBuffer; 822 final private int mPixelStride; 823 final private int mRowStride; 824 825 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 826 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)827 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 828 mRowStride = rowStride; 829 mPixelStride = pixelStride; 830 mBuffer = buffer; 831 /** 832 * Set the byteBuffer order according to host endianness (native 833 * order), otherwise, the byteBuffer order defaults to 834 * ByteOrder.BIG_ENDIAN. 835 */ 836 mBuffer.order(ByteOrder.nativeOrder()); 837 } 838 839 @Override getRowStride()840 public int getRowStride() { 841 throwISEIfImageIsInvalid(); 842 return mRowStride; 843 } 844 845 @Override getPixelStride()846 public int getPixelStride() { 847 throwISEIfImageIsInvalid(); 848 return mPixelStride; 849 } 850 851 @Override getBuffer()852 public ByteBuffer getBuffer() { 853 throwISEIfImageIsInvalid(); 854 return mBuffer; 855 } 856 clearBuffer()857 private void clearBuffer() { 858 // Need null check first, as the getBuffer() may not be called 859 // before an Image is closed. 860 if (mBuffer == null) { 861 return; 862 } 863 864 if (mBuffer.isDirect()) { 865 NioUtils.freeDirectBuffer(mBuffer); 866 } 867 mBuffer = null; 868 } 869 870 } 871 872 // Create the SurfacePlane object and fill the information nativeCreatePlanes(int numPlanes, int writerFmt)873 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt); 874 nativeGetWidth()875 private synchronized native int nativeGetWidth(); 876 nativeGetHeight()877 private synchronized native int nativeGetHeight(); 878 nativeGetFormat()879 private synchronized native int nativeGetFormat(); 880 nativeGetHardwareBuffer()881 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 882 } 883 884 // Native implemented ImageWriter methods. nativeInit(Object weakSelf, Surface surface, int maxImgs, int format)885 private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs, 886 int format); 887 nativeClose(long nativeCtx)888 private synchronized native void nativeClose(long nativeCtx); 889 nativeDequeueInputImage(long nativeCtx, Image wi)890 private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi); 891 nativeQueueInputImage(long nativeCtx, Image image, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)892 private synchronized native void nativeQueueInputImage(long nativeCtx, Image image, 893 long timestampNs, int left, int top, int right, int bottom, int transform, 894 int scalingMode); 895 nativeAttachAndQueueImage(long nativeCtx, long imageNativeBuffer, int imageFormat, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)896 private synchronized native int nativeAttachAndQueueImage(long nativeCtx, 897 long imageNativeBuffer, int imageFormat, long timestampNs, int left, 898 int top, int right, int bottom, int transform, int scalingMode); 899 cancelImage(long nativeCtx, Image image)900 private synchronized native void cancelImage(long nativeCtx, Image image); 901 902 /** 903 * We use a class initializer to allow the native code to cache some field 904 * offsets. 905 */ nativeClassInit()906 private static native void nativeClassInit(); 907 908 static { 909 System.loadLibrary("media_jni"); nativeClassInit()910 nativeClassInit(); 911 } 912 } 913