1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.graphics.GraphicBuffer; 22 import android.graphics.ImageFormat; 23 import android.graphics.ImageFormat.Format; 24 import android.graphics.PixelFormat; 25 import android.graphics.Rect; 26 import android.hardware.camera2.params.StreamConfigurationMap; 27 import android.hardware.camera2.utils.SurfaceUtils; 28 import android.hardware.HardwareBuffer; 29 import android.os.Handler; 30 import android.os.Looper; 31 import android.os.Message; 32 import android.util.Size; 33 import android.view.Surface; 34 35 import dalvik.system.VMRuntime; 36 37 import java.lang.ref.WeakReference; 38 import java.nio.ByteBuffer; 39 import java.nio.ByteOrder; 40 import java.nio.NioUtils; 41 import java.util.List; 42 import java.util.concurrent.CopyOnWriteArrayList; 43 44 /** 45 * <p> 46 * The ImageWriter class allows an application to produce Image data into a 47 * {@link android.view.Surface}, and have it be consumed by another component 48 * like {@link android.hardware.camera2.CameraDevice CameraDevice}. 49 * </p> 50 * <p> 51 * Several Android API classes can provide input {@link android.view.Surface 52 * Surface} objects for ImageWriter to produce data into, including 53 * {@link MediaCodec MediaCodec} (encoder), 54 * {@link android.hardware.camera2.CameraCaptureSession CameraCaptureSession} 55 * (reprocessing input), {@link ImageReader}, etc. 56 * </p> 57 * <p> 58 * The input Image data is encapsulated in {@link Image} objects. To produce 59 * Image data into a destination {@link android.view.Surface Surface}, the 60 * application can get an input Image via {@link #dequeueInputImage} then write 61 * Image data into it. Multiple such {@link Image} objects can be dequeued at 62 * the same time and queued back in any order, up to the number specified by the 63 * {@code maxImages} constructor parameter. 64 * </p> 65 * <p> 66 * If the application already has an Image from {@link ImageReader}, the 67 * application can directly queue this Image into the ImageWriter (via 68 * {@link #queueInputImage}), potentially with zero buffer copies. This 69 * even works if the image format of the ImageWriter is 70 * {@link ImageFormat#PRIVATE PRIVATE}, and prior to Android P is the only 71 * way to enqueue images into such an ImageWriter. Starting in Android P 72 * private images may also be accessed through their hardware buffers 73 * (when available) through the {@link Image#getHardwareBuffer()} method. 74 * Attempting to access the planes of a private image, will return an 75 * empty array. 76 * </p> 77 * <p> 78 * Once new input Images are queued into an ImageWriter, it's up to the 79 * downstream components (e.g. {@link ImageReader} or 80 * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the 81 * downstream components cannot consume the Images at least as fast as the 82 * ImageWriter production rate, the {@link #dequeueInputImage} call will 83 * eventually block and the application will have to drop input frames. 84 * </p> 85 * <p> 86 * If the consumer component that provided the input {@link android.view.Surface Surface} 87 * abandons the {@link android.view.Surface Surface}, {@link #queueInputImage queueing} 88 * or {@link #dequeueInputImage dequeueing} an {@link Image} will throw an 89 * {@link IllegalStateException}. 90 * </p> 91 */ 92 public class ImageWriter implements AutoCloseable { 93 private final Object mListenerLock = new Object(); 94 private OnImageReleasedListener mListener; 95 private ListenerHandler mListenerHandler; 96 private long mNativeContext; 97 98 // Field below is used by native code, do not access or modify. 99 private int mWriterFormat; 100 101 private final int mMaxImages; 102 // Keep track of the currently dequeued Image. This need to be thread safe as the images 103 // could be closed by different threads (e.g., application thread and GC thread). 104 private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>(); 105 private int mEstimatedNativeAllocBytes; 106 107 /** 108 * <p> 109 * Create a new ImageWriter. 110 * </p> 111 * <p> 112 * The {@code maxImages} parameter determines the maximum number of 113 * {@link Image} objects that can be be dequeued from the 114 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 115 * more memory, so it is important to use only the minimum number necessary. 116 * </p> 117 * <p> 118 * The input Image size and format depend on the Surface that is provided by 119 * the downstream consumer end-point. 120 * </p> 121 * 122 * @param surface The destination Surface this writer produces Image data 123 * into. 124 * @param maxImages The maximum number of Images the user will want to 125 * access simultaneously for producing Image data. This should be 126 * as small as possible to limit memory use. Once maxImages 127 * Images are dequeued by the user, one of them has to be queued 128 * back before a new Image can be dequeued for access via 129 * {@link #dequeueInputImage()}. 130 * @return a new ImageWriter instance. 131 */ newInstance(@onNull Surface surface, @IntRange(from = 1) int maxImages)132 public static @NonNull ImageWriter newInstance(@NonNull Surface surface, 133 @IntRange(from = 1) int maxImages) { 134 return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN, -1 /*width*/, 135 -1 /*height*/); 136 } 137 138 /** 139 * <p> 140 * Create a new ImageWriter with given number of max Images, format and producer dimension. 141 * </p> 142 * <p> 143 * The {@code maxImages} parameter determines the maximum number of 144 * {@link Image} objects that can be be dequeued from the 145 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 146 * more memory, so it is important to use only the minimum number necessary. 147 * </p> 148 * <p> 149 * The format specifies the image format of this ImageWriter. The format 150 * from the {@code surface} will be overridden with this format. For example, 151 * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default 152 * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter 153 * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate 154 * with {@link ImageFormat#PRIVATE} Images. 155 * </p> 156 * <p> 157 * Note that the consumer end-point may or may not be able to support Images with different 158 * format, for such case, the application should only use this method if the consumer is able 159 * to consume such images. 160 * </p> 161 * <p> The input Image size can also be set by the client. </p> 162 * 163 * @param surface The destination Surface this writer produces Image data 164 * into. 165 * @param maxImages The maximum number of Images the user will want to 166 * access simultaneously for producing Image data. This should be 167 * as small as possible to limit memory use. Once maxImages 168 * Images are dequeued by the user, one of them has to be queued 169 * back before a new Image can be dequeued for access via 170 * {@link #dequeueInputImage()}. 171 * @param format The format of this ImageWriter. It can be any valid format specified by 172 * {@link ImageFormat} or {@link PixelFormat}. 173 * 174 * @param width Input size width. 175 * @param height Input size height. 176 * 177 * @return a new ImageWriter instance. 178 * 179 * @hide 180 */ newInstance(@onNull Surface surface, @IntRange(from = 1) int maxImages, @Format int format, int width, int height)181 public static @NonNull ImageWriter newInstance(@NonNull Surface surface, 182 @IntRange(from = 1) int maxImages, @Format int format, int width, int height) { 183 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 184 throw new IllegalArgumentException("Invalid format is specified: " + format); 185 } 186 return new ImageWriter(surface, maxImages, format, width, height); 187 } 188 189 /** 190 * <p> 191 * Create a new ImageWriter with given number of max Images and format. 192 * </p> 193 * <p> 194 * The {@code maxImages} parameter determines the maximum number of 195 * {@link Image} objects that can be be dequeued from the 196 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 197 * more memory, so it is important to use only the minimum number necessary. 198 * </p> 199 * <p> 200 * The format specifies the image format of this ImageWriter. The format 201 * from the {@code surface} will be overridden with this format. For example, 202 * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default 203 * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter 204 * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate 205 * with {@link ImageFormat#PRIVATE} Images. 206 * </p> 207 * <p> 208 * Note that the consumer end-point may or may not be able to support Images with different 209 * format, for such case, the application should only use this method if the consumer is able 210 * to consume such images. 211 * </p> 212 * <p> 213 * The input Image size depends on the Surface that is provided by 214 * the downstream consumer end-point. 215 * </p> 216 * 217 * @param surface The destination Surface this writer produces Image data 218 * into. 219 * @param maxImages The maximum number of Images the user will want to 220 * access simultaneously for producing Image data. This should be 221 * as small as possible to limit memory use. Once maxImages 222 * Images are dequeued by the user, one of them has to be queued 223 * back before a new Image can be dequeued for access via 224 * {@link #dequeueInputImage()}. 225 * @param format The format of this ImageWriter. It can be any valid format specified by 226 * {@link ImageFormat} or {@link PixelFormat}. 227 * 228 * @return a new ImageWriter instance. 229 */ newInstance(@onNull Surface surface, @IntRange(from = 1) int maxImages, @Format int format)230 public static @NonNull ImageWriter newInstance(@NonNull Surface surface, 231 @IntRange(from = 1) int maxImages, @Format int format) { 232 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 233 throw new IllegalArgumentException("Invalid format is specified: " + format); 234 } 235 return new ImageWriter(surface, maxImages, format, -1 /*width*/, -1 /*height*/); 236 } 237 238 /** 239 * @hide 240 */ ImageWriter(Surface surface, int maxImages, int format, int width, int height)241 protected ImageWriter(Surface surface, int maxImages, int format, int width, int height) { 242 if (surface == null || maxImages < 1) { 243 throw new IllegalArgumentException("Illegal input argument: surface " + surface 244 + ", maxImages: " + maxImages); 245 } 246 247 mMaxImages = maxImages; 248 249 // Note that the underlying BufferQueue is working in synchronous mode 250 // to avoid dropping any buffers. 251 mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format, width, 252 height); 253 254 // nativeInit internally overrides UNKNOWN format. So does surface format query after 255 // nativeInit and before getEstimatedNativeAllocBytes(). 256 if (format == ImageFormat.UNKNOWN) { 257 format = SurfaceUtils.getSurfaceFormat(surface); 258 } 259 // Several public formats use the same native HAL_PIXEL_FORMAT_BLOB. The native 260 // allocation estimation sequence depends on the public formats values. To avoid 261 // possible errors, convert where necessary. 262 if (format == StreamConfigurationMap.HAL_PIXEL_FORMAT_BLOB) { 263 int surfaceDataspace = SurfaceUtils.getSurfaceDataspace(surface); 264 switch (surfaceDataspace) { 265 case StreamConfigurationMap.HAL_DATASPACE_DEPTH: 266 format = ImageFormat.DEPTH_POINT_CLOUD; 267 break; 268 case StreamConfigurationMap.HAL_DATASPACE_DYNAMIC_DEPTH: 269 format = ImageFormat.DEPTH_JPEG; 270 break; 271 case StreamConfigurationMap.HAL_DATASPACE_HEIF: 272 format = ImageFormat.HEIC; 273 break; 274 default: 275 format = ImageFormat.JPEG; 276 } 277 } 278 // Estimate the native buffer allocation size and register it so it gets accounted for 279 // during GC. Note that this doesn't include the buffers required by the buffer queue 280 // itself and the buffers requested by the producer. 281 // Only include memory for 1 buffer, since actually accounting for the memory used is 282 // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some 283 // size. 284 Size surfSize = SurfaceUtils.getSurfaceSize(surface); 285 mEstimatedNativeAllocBytes = 286 ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(), 287 format, /*buffer count*/ 1); 288 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 289 } 290 291 /** 292 * <p> 293 * Maximum number of Images that can be dequeued from the ImageWriter 294 * simultaneously (for example, with {@link #dequeueInputImage()}). 295 * </p> 296 * <p> 297 * An Image is considered dequeued after it's returned by 298 * {@link #dequeueInputImage()} from ImageWriter, and until the Image is 299 * sent back to ImageWriter via {@link #queueInputImage}, or 300 * {@link Image#close()}. 301 * </p> 302 * <p> 303 * Attempting to dequeue more than {@code maxImages} concurrently will 304 * result in the {@link #dequeueInputImage()} function throwing an 305 * {@link IllegalStateException}. 306 * </p> 307 * 308 * @return Maximum number of Images that can be dequeued from this 309 * ImageWriter. 310 * @see #dequeueInputImage 311 * @see #queueInputImage 312 * @see Image#close 313 */ getMaxImages()314 public int getMaxImages() { 315 return mMaxImages; 316 } 317 318 /** 319 * <p> 320 * Dequeue the next available input Image for the application to produce 321 * data into. 322 * </p> 323 * <p> 324 * This method requests a new input Image from ImageWriter. The application 325 * owns this Image after this call. Once the application fills the Image 326 * data, it is expected to return this Image back to ImageWriter for 327 * downstream consumer components (e.g. 328 * {@link android.hardware.camera2.CameraDevice}) to consume. The Image can 329 * be returned to ImageWriter via {@link #queueInputImage} or 330 * {@link Image#close()}. 331 * </p> 332 * <p> 333 * This call will block if all available input images have been queued by 334 * the application and the downstream consumer has not yet consumed any. 335 * When an Image is consumed by the downstream consumer and released, an 336 * {@link OnImageReleasedListener#onImageReleased} callback will be fired, 337 * which indicates that there is one input Image available. For non- 338 * {@link ImageFormat#PRIVATE PRIVATE} formats ( 339 * {@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE}), it is 340 * recommended to dequeue the next Image only after this callback is fired, 341 * in the steady state. 342 * </p> 343 * <p> 344 * If the format of ImageWriter is {@link ImageFormat#PRIVATE PRIVATE} ( 345 * {@link ImageWriter#getFormat()} == {@link ImageFormat#PRIVATE}), the 346 * image buffer is accessible to the application only through the hardware 347 * buffer obtained through {@link Image#getHardwareBuffer()}. (On Android 348 * versions prior to P, dequeueing private buffers will cause an 349 * {@link IllegalStateException} to be thrown). Alternatively, 350 * the application can acquire images from some other component (e.g. an 351 * {@link ImageReader}), and queue them directly to this ImageWriter via the 352 * {@link ImageWriter#queueInputImage queueInputImage()} method. 353 * </p> 354 * 355 * @return The next available input Image from this ImageWriter. 356 * @throws IllegalStateException if {@code maxImages} Images are currently 357 * dequeued, or the input {@link android.view.Surface Surface} 358 * has been abandoned by the consumer component that provided 359 * the {@link android.view.Surface Surface}. Prior to Android 360 * P, throws if the ImageWriter format is 361 * {@link ImageFormat#PRIVATE PRIVATE}. 362 * @see #queueInputImage 363 * @see Image#close 364 */ dequeueInputImage()365 public Image dequeueInputImage() { 366 if (mDequeuedImages.size() >= mMaxImages) { 367 throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages); 368 } 369 WriterSurfaceImage newImage = new WriterSurfaceImage(this); 370 nativeDequeueInputImage(mNativeContext, newImage); 371 mDequeuedImages.add(newImage); 372 newImage.mIsImageValid = true; 373 return newImage; 374 } 375 376 /** 377 * <p> 378 * Queue an input {@link Image} back to ImageWriter for the downstream 379 * consumer to access. 380 * </p> 381 * <p> 382 * The input {@link Image} could be from ImageReader (acquired via 383 * {@link ImageReader#acquireNextImage} or 384 * {@link ImageReader#acquireLatestImage}), or from this ImageWriter 385 * (acquired via {@link #dequeueInputImage}). In the former case, the Image 386 * data will be moved to this ImageWriter. Note that the Image properties 387 * (size, format, strides, etc.) must be the same as the properties of the 388 * images dequeued from this ImageWriter. In the latter case, the application has 389 * filled the input image with data. This method then passes the filled 390 * buffer to the downstream consumer. In both cases, it's up to the caller 391 * to ensure that the Image timestamp (in nanoseconds) is correctly set, as 392 * the downstream component may want to use it to indicate the Image data 393 * capture time. 394 * </p> 395 * <p> 396 * After this method is called and the downstream consumer consumes and 397 * releases the Image, an {@link OnImageReleasedListener#onImageReleased} 398 * callback will fire. The application can use this callback to avoid 399 * sending Images faster than the downstream consumer processing rate in 400 * steady state. 401 * </p> 402 * <p> 403 * Passing in an Image from some other component (e.g. an 404 * {@link ImageReader}) requires a free input Image from this ImageWriter as 405 * the destination. In this case, this call will block, as 406 * {@link #dequeueInputImage} does, if there are no free Images available. 407 * To avoid blocking, the application should ensure that there is at least 408 * one free Image available in this ImageWriter before calling this method. 409 * </p> 410 * <p> 411 * After this call, the input Image is no longer valid for further access, 412 * as if the Image is {@link Image#close closed}. Attempting to access the 413 * {@link ByteBuffer ByteBuffers} returned by an earlier 414 * {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an 415 * {@link IllegalStateException}. 416 * </p> 417 * 418 * @param image The Image to be queued back to ImageWriter for future 419 * consumption. 420 * @throws IllegalStateException if the image was already queued previously, 421 * or the image was aborted previously, or the input 422 * {@link android.view.Surface Surface} has been abandoned by the 423 * consumer component that provided the 424 * {@link android.view.Surface Surface}. 425 * @see #dequeueInputImage() 426 */ queueInputImage(Image image)427 public void queueInputImage(Image image) { 428 if (image == null) { 429 throw new IllegalArgumentException("image shouldn't be null"); 430 } 431 boolean ownedByMe = isImageOwnedByMe(image); 432 if (ownedByMe && !(((WriterSurfaceImage) image).mIsImageValid)) { 433 throw new IllegalStateException("Image from ImageWriter is invalid"); 434 } 435 436 // For images from other components that have non-null owner, need to detach first, 437 // then attach. Images without owners must already be attachable. 438 if (!ownedByMe) { 439 if ((image.getOwner() instanceof ImageReader)) { 440 ImageReader prevOwner = (ImageReader) image.getOwner(); 441 442 prevOwner.detachImage(image); 443 } else if (image.getOwner() != null) { 444 throw new IllegalArgumentException("Only images from ImageReader can be queued to" 445 + " ImageWriter, other image source is not supported yet!"); 446 } 447 448 attachAndQueueInputImage(image); 449 // This clears the native reference held by the original owner. 450 // When this Image is detached later by this ImageWriter, the 451 // native memory won't be leaked. 452 image.close(); 453 return; 454 } 455 456 Rect crop = image.getCropRect(); 457 nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top, 458 crop.right, crop.bottom, image.getTransform(), image.getScalingMode()); 459 460 /** 461 * Only remove and cleanup the Images that are owned by this 462 * ImageWriter. Images detached from other owners are only temporarily 463 * owned by this ImageWriter and will be detached immediately after they 464 * are released by downstream consumers, so there is no need to keep 465 * track of them in mDequeuedImages. 466 */ 467 if (ownedByMe) { 468 mDequeuedImages.remove(image); 469 // Do not call close here, as close is essentially cancel image. 470 WriterSurfaceImage wi = (WriterSurfaceImage) image; 471 wi.clearSurfacePlanes(); 472 wi.mIsImageValid = false; 473 } 474 } 475 476 /** 477 * Get the ImageWriter format. 478 * <p> 479 * This format may be different than the Image format returned by 480 * {@link Image#getFormat()}. However, if the ImageWriter format is 481 * {@link ImageFormat#PRIVATE PRIVATE}, calling {@link #dequeueInputImage()} 482 * will result in an {@link IllegalStateException}. 483 * </p> 484 * 485 * @return The ImageWriter format. 486 */ getFormat()487 public int getFormat() { 488 return mWriterFormat; 489 } 490 491 /** 492 * ImageWriter callback interface, used to to asynchronously notify the 493 * application of various ImageWriter events. 494 */ 495 public interface OnImageReleasedListener { 496 /** 497 * <p> 498 * Callback that is called when an input Image is released back to 499 * ImageWriter after the data consumption. 500 * </p> 501 * <p> 502 * The client can use this callback to be notified that an input Image 503 * has been consumed and released by the downstream consumer. More 504 * specifically, this callback will be fired for below cases: 505 * <li>The application dequeues an input Image via the 506 * {@link ImageWriter#dequeueInputImage dequeueInputImage()} method, 507 * uses it, and then queues it back to this ImageWriter via the 508 * {@link ImageWriter#queueInputImage queueInputImage()} method. After 509 * the downstream consumer uses and releases this image to this 510 * ImageWriter, this callback will be fired. This image will be 511 * available to be dequeued after this callback.</li> 512 * <li>The application obtains an Image from some other component (e.g. 513 * an {@link ImageReader}), uses it, and then queues it to this 514 * ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}. 515 * After the downstream consumer uses and releases this image to this 516 * ImageWriter, this callback will be fired.</li> 517 * </p> 518 * 519 * @param writer the ImageWriter the callback is associated with. 520 * @see ImageWriter 521 * @see Image 522 */ onImageReleased(ImageWriter writer)523 void onImageReleased(ImageWriter writer); 524 } 525 526 /** 527 * Register a listener to be invoked when an input Image is returned to the 528 * ImageWriter. 529 * 530 * @param listener The listener that will be run. 531 * @param handler The handler on which the listener should be invoked, or 532 * null if the listener should be invoked on the calling thread's 533 * looper. 534 * @throws IllegalArgumentException If no handler specified and the calling 535 * thread has no looper. 536 */ setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler)537 public void setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler) { 538 synchronized (mListenerLock) { 539 if (listener != null) { 540 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 541 if (looper == null) { 542 throw new IllegalArgumentException( 543 "handler is null but the current thread is not a looper"); 544 } 545 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 546 mListenerHandler = new ListenerHandler(looper); 547 } 548 mListener = listener; 549 } else { 550 mListener = null; 551 mListenerHandler = null; 552 } 553 } 554 } 555 556 /** 557 * Free up all the resources associated with this ImageWriter. 558 * <p> 559 * After calling this method, this ImageWriter cannot be used. Calling any 560 * methods on this ImageWriter and Images previously provided by 561 * {@link #dequeueInputImage()} will result in an 562 * {@link IllegalStateException}, and attempting to write into 563 * {@link ByteBuffer ByteBuffers} returned by an earlier 564 * {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined 565 * behavior. 566 * </p> 567 */ 568 @Override close()569 public void close() { 570 setOnImageReleasedListener(null, null); 571 for (Image image : mDequeuedImages) { 572 image.close(); 573 } 574 mDequeuedImages.clear(); 575 nativeClose(mNativeContext); 576 mNativeContext = 0; 577 578 if (mEstimatedNativeAllocBytes > 0) { 579 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 580 mEstimatedNativeAllocBytes = 0; 581 } 582 } 583 584 @Override finalize()585 protected void finalize() throws Throwable { 586 try { 587 close(); 588 } finally { 589 super.finalize(); 590 } 591 } 592 593 /** 594 * <p> 595 * Attach and queue input Image to this ImageWriter. 596 * </p> 597 * <p> 598 * When the format of an Image is {@link ImageFormat#PRIVATE PRIVATE}, or 599 * the source Image is so large that copying its data is too expensive, this 600 * method can be used to migrate the source Image into ImageWriter without a 601 * data copy, and then queue it to this ImageWriter. The source Image must 602 * be detached from its previous owner already, or this call will throw an 603 * {@link IllegalStateException}. 604 * </p> 605 * <p> 606 * After this call, the ImageWriter takes ownership of this Image. This 607 * ownership will automatically be removed from this writer after the 608 * consumer releases this Image, that is, after 609 * {@link OnImageReleasedListener#onImageReleased}. The caller is responsible for 610 * closing this Image through {@link Image#close()} to free up the resources 611 * held by this Image. 612 * </p> 613 * 614 * @param image The source Image to be attached and queued into this 615 * ImageWriter for downstream consumer to use. 616 * @throws IllegalStateException if the Image is not detached from its 617 * previous owner, or the Image is already attached to this 618 * ImageWriter, or the source Image is invalid. 619 */ attachAndQueueInputImage(Image image)620 private void attachAndQueueInputImage(Image image) { 621 if (image == null) { 622 throw new IllegalArgumentException("image shouldn't be null"); 623 } 624 if (isImageOwnedByMe(image)) { 625 throw new IllegalArgumentException( 626 "Can not attach an image that is owned ImageWriter already"); 627 } 628 /** 629 * Throw ISE if the image is not attachable, which means that it is 630 * either owned by other entity now, or completely non-attachable (some 631 * stand-alone images are not backed by native gralloc buffer, thus not 632 * attachable). 633 */ 634 if (!image.isAttachable()) { 635 throw new IllegalStateException("Image was not detached from last owner, or image " 636 + " is not detachable"); 637 } 638 639 // TODO: what if attach failed, throw RTE or detach a slot then attach? 640 // need do some cleanup to make sure no orphaned 641 // buffer caused leak. 642 Rect crop = image.getCropRect(); 643 if (image.getNativeContext() != 0) { 644 nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), 645 image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, 646 image.getTransform(), image.getScalingMode()); 647 } else { 648 GraphicBuffer gb = GraphicBuffer.createFromHardwareBuffer(image.getHardwareBuffer()); 649 nativeAttachAndQueueGraphicBuffer(mNativeContext, gb, image.getFormat(), 650 image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, 651 image.getTransform(), image.getScalingMode()); 652 gb.destroy(); 653 image.close(); 654 } 655 } 656 657 /** 658 * This custom handler runs asynchronously so callbacks don't get queued 659 * behind UI messages. 660 */ 661 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)662 public ListenerHandler(Looper looper) { 663 super(looper, null, true /* async */); 664 } 665 666 @Override handleMessage(Message msg)667 public void handleMessage(Message msg) { 668 OnImageReleasedListener listener; 669 synchronized (mListenerLock) { 670 listener = mListener; 671 } 672 if (listener != null) { 673 listener.onImageReleased(ImageWriter.this); 674 } 675 } 676 } 677 678 /** 679 * Called from Native code when an Event happens. This may be called from an 680 * arbitrary Binder thread, so access to the ImageWriter must be 681 * synchronized appropriately. 682 */ postEventFromNative(Object selfRef)683 private static void postEventFromNative(Object selfRef) { 684 @SuppressWarnings("unchecked") 685 WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef; 686 final ImageWriter iw = weakSelf.get(); 687 if (iw == null) { 688 return; 689 } 690 691 final Handler handler; 692 synchronized (iw.mListenerLock) { 693 handler = iw.mListenerHandler; 694 } 695 if (handler != null) { 696 handler.sendEmptyMessage(0); 697 } 698 } 699 700 /** 701 * <p> 702 * Abort the Images that were dequeued from this ImageWriter, and return 703 * them to this writer for reuse. 704 * </p> 705 * <p> 706 * This method is used for the cases where the application dequeued the 707 * Image, may have filled the data, but does not want the downstream 708 * component to consume it. The Image will be returned to this ImageWriter 709 * for reuse after this call, and the ImageWriter will immediately have an 710 * Image available to be dequeued. This aborted Image will be invisible to 711 * the downstream consumer, as if nothing happened. 712 * </p> 713 * 714 * @param image The Image to be aborted. 715 * @see #dequeueInputImage() 716 * @see Image#close() 717 */ abortImage(Image image)718 private void abortImage(Image image) { 719 if (image == null) { 720 throw new IllegalArgumentException("image shouldn't be null"); 721 } 722 723 if (!mDequeuedImages.contains(image)) { 724 throw new IllegalStateException("It is illegal to abort some image that is not" 725 + " dequeued yet"); 726 } 727 728 WriterSurfaceImage wi = (WriterSurfaceImage) image; 729 if (!wi.mIsImageValid) { 730 return; 731 } 732 733 /** 734 * We only need abort Images that are owned and dequeued by ImageWriter. 735 * For attached Images, no need to abort, as there are only two cases: 736 * attached + queued successfully, and attach failed. Neither of the 737 * cases need abort. 738 */ 739 cancelImage(mNativeContext, image); 740 mDequeuedImages.remove(image); 741 wi.clearSurfacePlanes(); 742 wi.mIsImageValid = false; 743 } 744 isImageOwnedByMe(Image image)745 private boolean isImageOwnedByMe(Image image) { 746 if (!(image instanceof WriterSurfaceImage)) { 747 return false; 748 } 749 WriterSurfaceImage wi = (WriterSurfaceImage) image; 750 if (wi.getOwner() != this) { 751 return false; 752 } 753 754 return true; 755 } 756 757 private static class WriterSurfaceImage extends android.media.Image { 758 private ImageWriter mOwner; 759 // This field is used by native code, do not access or modify. 760 private long mNativeBuffer; 761 private int mNativeFenceFd = -1; 762 private SurfacePlane[] mPlanes; 763 private int mHeight = -1; 764 private int mWidth = -1; 765 private int mFormat = -1; 766 // When this default timestamp is used, timestamp for the input Image 767 // will be generated automatically when queueInputBuffer is called. 768 private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE; 769 private long mTimestamp = DEFAULT_TIMESTAMP; 770 771 private int mTransform = 0; //Default no transform 772 private int mScalingMode = 0; //Default frozen scaling mode 773 WriterSurfaceImage(ImageWriter writer)774 public WriterSurfaceImage(ImageWriter writer) { 775 mOwner = writer; 776 } 777 778 @Override getFormat()779 public int getFormat() { 780 throwISEIfImageIsInvalid(); 781 782 if (mFormat == -1) { 783 mFormat = nativeGetFormat(); 784 } 785 return mFormat; 786 } 787 788 @Override getWidth()789 public int getWidth() { 790 throwISEIfImageIsInvalid(); 791 792 if (mWidth == -1) { 793 mWidth = nativeGetWidth(); 794 } 795 796 return mWidth; 797 } 798 799 @Override getHeight()800 public int getHeight() { 801 throwISEIfImageIsInvalid(); 802 803 if (mHeight == -1) { 804 mHeight = nativeGetHeight(); 805 } 806 807 return mHeight; 808 } 809 810 @Override getTransform()811 public int getTransform() { 812 throwISEIfImageIsInvalid(); 813 814 return mTransform; 815 } 816 817 @Override getScalingMode()818 public int getScalingMode() { 819 throwISEIfImageIsInvalid(); 820 821 return mScalingMode; 822 } 823 824 @Override getTimestamp()825 public long getTimestamp() { 826 throwISEIfImageIsInvalid(); 827 828 return mTimestamp; 829 } 830 831 @Override setTimestamp(long timestamp)832 public void setTimestamp(long timestamp) { 833 throwISEIfImageIsInvalid(); 834 835 mTimestamp = timestamp; 836 } 837 838 @Override getHardwareBuffer()839 public HardwareBuffer getHardwareBuffer() { 840 throwISEIfImageIsInvalid(); 841 842 return nativeGetHardwareBuffer(); 843 } 844 845 @Override getPlanes()846 public Plane[] getPlanes() { 847 throwISEIfImageIsInvalid(); 848 849 if (mPlanes == null) { 850 int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat()); 851 mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat()); 852 } 853 854 return mPlanes.clone(); 855 } 856 857 @Override isAttachable()858 public boolean isAttachable() { 859 throwISEIfImageIsInvalid(); 860 // Don't allow Image to be detached from ImageWriter for now, as no 861 // detach API is exposed. 862 return false; 863 } 864 865 @Override getOwner()866 ImageWriter getOwner() { 867 throwISEIfImageIsInvalid(); 868 869 return mOwner; 870 } 871 872 @Override getNativeContext()873 long getNativeContext() { 874 throwISEIfImageIsInvalid(); 875 876 return mNativeBuffer; 877 } 878 879 @Override close()880 public void close() { 881 if (mIsImageValid) { 882 getOwner().abortImage(this); 883 } 884 } 885 886 @Override finalize()887 protected final void finalize() throws Throwable { 888 try { 889 close(); 890 } finally { 891 super.finalize(); 892 } 893 } 894 clearSurfacePlanes()895 private void clearSurfacePlanes() { 896 if (mIsImageValid && mPlanes != null) { 897 for (int i = 0; i < mPlanes.length; i++) { 898 if (mPlanes[i] != null) { 899 mPlanes[i].clearBuffer(); 900 mPlanes[i] = null; 901 } 902 } 903 } 904 } 905 906 private class SurfacePlane extends android.media.Image.Plane { 907 private ByteBuffer mBuffer; 908 final private int mPixelStride; 909 final private int mRowStride; 910 911 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 912 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)913 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 914 mRowStride = rowStride; 915 mPixelStride = pixelStride; 916 mBuffer = buffer; 917 /** 918 * Set the byteBuffer order according to host endianness (native 919 * order), otherwise, the byteBuffer order defaults to 920 * ByteOrder.BIG_ENDIAN. 921 */ 922 mBuffer.order(ByteOrder.nativeOrder()); 923 } 924 925 @Override getRowStride()926 public int getRowStride() { 927 throwISEIfImageIsInvalid(); 928 return mRowStride; 929 } 930 931 @Override getPixelStride()932 public int getPixelStride() { 933 throwISEIfImageIsInvalid(); 934 return mPixelStride; 935 } 936 937 @Override getBuffer()938 public ByteBuffer getBuffer() { 939 throwISEIfImageIsInvalid(); 940 return mBuffer; 941 } 942 clearBuffer()943 private void clearBuffer() { 944 // Need null check first, as the getBuffer() may not be called 945 // before an Image is closed. 946 if (mBuffer == null) { 947 return; 948 } 949 950 if (mBuffer.isDirect()) { 951 NioUtils.freeDirectBuffer(mBuffer); 952 } 953 mBuffer = null; 954 } 955 956 } 957 958 // Create the SurfacePlane object and fill the information nativeCreatePlanes(int numPlanes, int writerFmt)959 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt); 960 nativeGetWidth()961 private synchronized native int nativeGetWidth(); 962 nativeGetHeight()963 private synchronized native int nativeGetHeight(); 964 nativeGetFormat()965 private synchronized native int nativeGetFormat(); 966 nativeGetHardwareBuffer()967 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 968 } 969 970 // Native implemented ImageWriter methods. nativeInit(Object weakSelf, Surface surface, int maxImgs, int format, int width, int height)971 private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs, 972 int format, int width, int height); 973 nativeClose(long nativeCtx)974 private synchronized native void nativeClose(long nativeCtx); 975 nativeDequeueInputImage(long nativeCtx, Image wi)976 private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi); 977 nativeQueueInputImage(long nativeCtx, Image image, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)978 private synchronized native void nativeQueueInputImage(long nativeCtx, Image image, 979 long timestampNs, int left, int top, int right, int bottom, int transform, 980 int scalingMode); 981 nativeAttachAndQueueImage(long nativeCtx, long imageNativeBuffer, int imageFormat, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)982 private synchronized native int nativeAttachAndQueueImage(long nativeCtx, 983 long imageNativeBuffer, int imageFormat, long timestampNs, int left, 984 int top, int right, int bottom, int transform, int scalingMode); nativeAttachAndQueueGraphicBuffer(long nativeCtx, GraphicBuffer graphicBuffer, int imageFormat, long timestampNs, int left, int top, int right, int bottom, int transform, int scalingMode)985 private synchronized native int nativeAttachAndQueueGraphicBuffer(long nativeCtx, 986 GraphicBuffer graphicBuffer, int imageFormat, long timestampNs, int left, 987 int top, int right, int bottom, int transform, int scalingMode); 988 cancelImage(long nativeCtx, Image image)989 private synchronized native void cancelImage(long nativeCtx, Image image); 990 991 /** 992 * We use a class initializer to allow the native code to cache some field 993 * offsets. 994 */ nativeClassInit()995 private static native void nativeClassInit(); 996 997 static { 998 System.loadLibrary("media_jni"); nativeClassInit()999 nativeClassInit(); 1000 } 1001 } 1002