1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.graphics.ImageFormat; 20 import android.graphics.PixelFormat; 21 import android.hardware.HardwareBuffer; 22 import android.media.Image.Plane; 23 import android.util.Log; 24 import android.util.Size; 25 26 import com.android.internal.camera.flags.Flags; 27 28 import libcore.io.Memory; 29 30 import java.nio.ByteBuffer; 31 32 /** 33 * Package private utility class for hosting commonly used Image related methods. 34 */ 35 class ImageUtils { 36 private static final String IMAGEUTILS_LOG_TAG = "ImageUtils"; 37 38 /** 39 * Only a subset of the formats defined in 40 * {@link android.graphics.ImageFormat ImageFormat} and 41 * {@link android.graphics.PixelFormat PixelFormat} are supported by 42 * ImageReader. When reading RGB data from a surface, the formats defined in 43 * {@link android.graphics.PixelFormat PixelFormat} can be used; when 44 * reading YUV, JPEG, HEIC or raw sensor data (for example, from the camera 45 * or video decoder), formats from {@link android.graphics.ImageFormat ImageFormat} 46 * are used. 47 */ getNumPlanesForFormat(int format)48 public static int getNumPlanesForFormat(int format) { 49 if (Flags.cameraHeifGainmap()) { 50 if (format == ImageFormat.HEIC_ULTRAHDR) { 51 return 1; 52 } 53 } 54 switch (format) { 55 case ImageFormat.YV12: 56 case ImageFormat.YUV_420_888: 57 case ImageFormat.NV21: 58 case ImageFormat.YCBCR_P010: 59 case ImageFormat.YCBCR_P210: 60 return 3; 61 case ImageFormat.NV16: 62 return 2; 63 case PixelFormat.RGB_565: 64 case PixelFormat.RGBA_8888: 65 case PixelFormat.RGBX_8888: 66 case PixelFormat.RGB_888: 67 case ImageFormat.JPEG: 68 case ImageFormat.YUY2: 69 case ImageFormat.Y8: 70 case ImageFormat.Y16: 71 case ImageFormat.RAW_SENSOR: 72 case ImageFormat.RAW_PRIVATE: 73 case ImageFormat.RAW10: 74 case ImageFormat.RAW12: 75 case ImageFormat.DEPTH16: 76 case ImageFormat.DEPTH_POINT_CLOUD: 77 case ImageFormat.RAW_DEPTH: 78 case ImageFormat.RAW_DEPTH10: 79 case ImageFormat.DEPTH_JPEG: 80 case ImageFormat.HEIC: 81 case ImageFormat.JPEG_R: 82 return 1; 83 case ImageFormat.PRIVATE: 84 return 0; 85 default: 86 throw new UnsupportedOperationException( 87 String.format("Invalid format specified %d", format)); 88 } 89 } 90 91 /** 92 * Only a subset of the formats defined in 93 * {@link android.graphics.HardwareBuffer.Format} constants are supported by ImageReader. 94 */ getNumPlanesForHardwareBufferFormat(int hardwareBufferFormat)95 public static int getNumPlanesForHardwareBufferFormat(int hardwareBufferFormat) { 96 switch(hardwareBufferFormat) { 97 case HardwareBuffer.YCBCR_420_888: 98 case HardwareBuffer.YCBCR_P010: 99 case HardwareBuffer.YCBCR_P210: 100 return 3; 101 case HardwareBuffer.RGBA_8888: 102 case HardwareBuffer.RGBX_8888: 103 case HardwareBuffer.RGB_888: 104 case HardwareBuffer.RGB_565: 105 case HardwareBuffer.RGBA_FP16: 106 case HardwareBuffer.RGBA_1010102: 107 case HardwareBuffer.BLOB: 108 case HardwareBuffer.D_16: 109 case HardwareBuffer.D_24: 110 case HardwareBuffer.DS_24UI8: 111 case HardwareBuffer.D_FP32: 112 case HardwareBuffer.DS_FP32UI8: 113 case HardwareBuffer.S_UI8: 114 return 1; 115 default: 116 throw new UnsupportedOperationException( 117 String.format("Invalid hardwareBuffer format specified %d", 118 hardwareBufferFormat)); 119 } 120 } 121 /** 122 * <p> 123 * Copy source image data to destination Image. 124 * </p> 125 * <p> 126 * Only support the copy between two non-{@link ImageFormat#PRIVATE PRIVATE} format 127 * images with same properties (format, size, etc.). The data from the 128 * source image will be copied to the byteBuffers from the destination Image 129 * starting from position zero, and the destination image will be rewound to 130 * zero after copy is done. 131 * </p> 132 * 133 * @param src The source image to be copied from. 134 * @param dst The destination image to be copied to. 135 * @throws IllegalArgumentException If the source and destination images 136 * have different format, or one of the images is not copyable. 137 */ imageCopy(Image src, Image dst)138 public static void imageCopy(Image src, Image dst) { 139 if (src == null || dst == null) { 140 throw new IllegalArgumentException("Images should be non-null"); 141 } 142 if (src.getFormat() != dst.getFormat()) { 143 throw new IllegalArgumentException("Src and dst images should have the same format"); 144 } 145 if (src.getFormat() == ImageFormat.PRIVATE || 146 dst.getFormat() == ImageFormat.PRIVATE) { 147 throw new IllegalArgumentException("PRIVATE format images are not copyable"); 148 } 149 if (src.getFormat() == ImageFormat.RAW_PRIVATE) { 150 throw new IllegalArgumentException( 151 "Copy of RAW_OPAQUE format has not been implemented"); 152 } 153 if (src.getFormat() == ImageFormat.RAW_DEPTH) { 154 throw new IllegalArgumentException( 155 "Copy of RAW_DEPTH format has not been implemented"); 156 } 157 if (src.getFormat() == ImageFormat.RAW_DEPTH10) { 158 throw new IllegalArgumentException( 159 "Copy of RAW_DEPTH10 format has not been implemented"); 160 } 161 if (!(dst.getOwner() instanceof ImageWriter)) { 162 throw new IllegalArgumentException("Destination image is not from ImageWriter. Only" 163 + " the images from ImageWriter are writable"); 164 } 165 Size srcSize = new Size(src.getWidth(), src.getHeight()); 166 Size dstSize = new Size(dst.getWidth(), dst.getHeight()); 167 if (!srcSize.equals(dstSize)) { 168 throw new IllegalArgumentException("source image size " + srcSize + " is different" 169 + " with " + "destination image size " + dstSize); 170 } 171 172 Plane[] srcPlanes = src.getPlanes(); 173 Plane[] dstPlanes = dst.getPlanes(); 174 ByteBuffer srcBuffer = null; 175 ByteBuffer dstBuffer = null; 176 for (int i = 0; i < srcPlanes.length; i++) { 177 int srcRowStride = srcPlanes[i].getRowStride(); 178 int dstRowStride = dstPlanes[i].getRowStride(); 179 srcBuffer = srcPlanes[i].getBuffer(); 180 dstBuffer = dstPlanes[i].getBuffer(); 181 if (!(srcBuffer.isDirect() && dstBuffer.isDirect())) { 182 throw new IllegalArgumentException("Source and destination ByteBuffers must be" 183 + " direct byteBuffer!"); 184 } 185 if (srcPlanes[i].getPixelStride() != dstPlanes[i].getPixelStride()) { 186 throw new IllegalArgumentException("Source plane image pixel stride " + 187 srcPlanes[i].getPixelStride() + 188 " must be same as destination image pixel stride " + 189 dstPlanes[i].getPixelStride()); 190 } 191 192 int srcPos = srcBuffer.position(); 193 srcBuffer.rewind(); 194 dstBuffer.rewind(); 195 if (srcRowStride == dstRowStride) { 196 // Fast path, just copy the content if the byteBuffer all together. 197 dstBuffer.put(srcBuffer); 198 } else { 199 // Source and destination images may have different alignment requirements, 200 // therefore may have different strides. Copy row by row for such case. 201 int srcOffset = srcBuffer.position(); 202 int dstOffset = dstBuffer.position(); 203 Size effectivePlaneSize = getEffectivePlaneSizeForImage(src, i); 204 int srcByteCount = effectivePlaneSize.getWidth() * srcPlanes[i].getPixelStride(); 205 for (int row = 0; row < effectivePlaneSize.getHeight(); row++) { 206 if (row == effectivePlaneSize.getHeight() - 1) { 207 // Special case for NV21 backed YUV420_888: need handle the last row 208 // carefully to avoid memory corruption. Check if we have enough bytes to 209 // copy. 210 int remainingBytes = srcBuffer.remaining() - srcOffset; 211 if (srcByteCount > remainingBytes) { 212 srcByteCount = remainingBytes; 213 } 214 } 215 directByteBufferCopy(srcBuffer, srcOffset, dstBuffer, dstOffset, srcByteCount); 216 srcOffset += srcRowStride; 217 dstOffset += dstRowStride; 218 } 219 } 220 221 srcBuffer.position(srcPos); 222 dstBuffer.rewind(); 223 } 224 } 225 226 /** 227 * Return the estimated native allocation size in bytes based on width, height, format, 228 * and number of images. 229 * 230 * <p>This is a very rough estimation and should only be used for native allocation 231 * registration in VM so it can be accounted for during GC.</p> 232 * 233 * @param width The width of the images. 234 * @param height The height of the images. 235 * @param format The format of the images. 236 * @param numImages The number of the images. 237 */ getEstimatedNativeAllocBytes(int width, int height, int format, int numImages)238 public static int getEstimatedNativeAllocBytes(int width, int height, int format, 239 int numImages) { 240 double estimatedBytePerPixel; 241 if (Flags.cameraHeifGainmap()) { 242 if (format == ImageFormat.HEIC_ULTRAHDR) { 243 estimatedBytePerPixel = 0.3; 244 } 245 } 246 switch (format) { 247 // 10x compression from RGB_888 248 case ImageFormat.JPEG: 249 case ImageFormat.DEPTH_POINT_CLOUD: 250 case ImageFormat.DEPTH_JPEG: 251 case ImageFormat.HEIC: 252 case ImageFormat.JPEG_R: 253 estimatedBytePerPixel = 0.3; 254 break; 255 case ImageFormat.Y8: 256 estimatedBytePerPixel = 1.0; 257 break; 258 case ImageFormat.RAW10: 259 case ImageFormat.RAW_DEPTH10: 260 estimatedBytePerPixel = 1.25; 261 break; 262 case ImageFormat.YV12: 263 case ImageFormat.YUV_420_888: 264 case ImageFormat.NV21: 265 case ImageFormat.RAW12: 266 case ImageFormat.PRIVATE: // A rough estimate because the real size is unknown. 267 estimatedBytePerPixel = 1.5; 268 break; 269 case ImageFormat.NV16: 270 case PixelFormat.RGB_565: 271 case ImageFormat.YUY2: 272 case ImageFormat.Y16: 273 case ImageFormat.RAW_DEPTH: 274 case ImageFormat.RAW_SENSOR: 275 case ImageFormat.RAW_PRIVATE: // round estimate, real size is unknown 276 case ImageFormat.DEPTH16: 277 estimatedBytePerPixel = 2.0; 278 break; 279 case PixelFormat.RGB_888: 280 case ImageFormat.YCBCR_P010: 281 estimatedBytePerPixel = 3.0; 282 break; 283 case PixelFormat.RGBA_8888: 284 case PixelFormat.RGBX_8888: 285 case PixelFormat.RGBA_1010102: 286 case ImageFormat.YCBCR_P210: 287 estimatedBytePerPixel = 4.0; 288 break; 289 default: 290 if (Log.isLoggable(IMAGEUTILS_LOG_TAG, Log.VERBOSE)) { 291 Log.v(IMAGEUTILS_LOG_TAG, "getEstimatedNativeAllocBytes() uses default" 292 + "estimated native allocation size."); 293 } 294 estimatedBytePerPixel = 1.0; 295 } 296 297 return (int)(width * height * estimatedBytePerPixel * numImages); 298 } 299 getEffectivePlaneSizeForImage(Image image, int planeIdx)300 private static Size getEffectivePlaneSizeForImage(Image image, int planeIdx) { 301 if (Flags.cameraHeifGainmap()) { 302 if (image.getFormat() == ImageFormat.HEIC_ULTRAHDR){ 303 return new Size(image.getWidth(), image.getHeight()); 304 } 305 } 306 switch (image.getFormat()) { 307 case ImageFormat.YCBCR_P010: 308 case ImageFormat.YV12: 309 case ImageFormat.YUV_420_888: 310 case ImageFormat.NV21: 311 if (planeIdx == 0) { 312 return new Size(image.getWidth(), image.getHeight()); 313 } else { 314 return new Size(image.getWidth() / 2, image.getHeight() / 2); 315 } 316 case ImageFormat.NV16: 317 if (planeIdx == 0) { 318 return new Size(image.getWidth(), image.getHeight()); 319 } else { 320 return new Size(image.getWidth(), image.getHeight() / 2); 321 } 322 case PixelFormat.RGB_565: 323 case PixelFormat.RGBA_8888: 324 case PixelFormat.RGBA_1010102: 325 case PixelFormat.RGBX_8888: 326 case PixelFormat.RGB_888: 327 case ImageFormat.JPEG: 328 case ImageFormat.YUY2: 329 case ImageFormat.Y8: 330 case ImageFormat.Y16: 331 case ImageFormat.RAW_SENSOR: 332 case ImageFormat.RAW10: 333 case ImageFormat.RAW12: 334 case ImageFormat.RAW_DEPTH: 335 case ImageFormat.RAW_DEPTH10: 336 case ImageFormat.HEIC: 337 case ImageFormat.JPEG_R: 338 return new Size(image.getWidth(), image.getHeight()); 339 case ImageFormat.PRIVATE: 340 return new Size(0, 0); 341 case ImageFormat.YCBCR_P210: 342 if (planeIdx == 0) { 343 return new Size(image.getWidth(), image.getHeight()); 344 } else { 345 return new Size(image.getWidth() / 2, image.getHeight()); 346 } 347 default: 348 if (Log.isLoggable(IMAGEUTILS_LOG_TAG, Log.VERBOSE)) { 349 Log.v(IMAGEUTILS_LOG_TAG, "getEffectivePlaneSizeForImage() uses" 350 + "image's width and height for plane size."); 351 } 352 return new Size(image.getWidth(), image.getHeight()); 353 } 354 } 355 directByteBufferCopy(ByteBuffer srcBuffer, int srcOffset, ByteBuffer dstBuffer, int dstOffset, int srcByteCount)356 private static void directByteBufferCopy(ByteBuffer srcBuffer, int srcOffset, 357 ByteBuffer dstBuffer, int dstOffset, int srcByteCount) { 358 Memory.memmove(dstBuffer, dstOffset, srcBuffer, srcOffset, srcByteCount); 359 } 360 } 361