1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.hardware.camera2.extension.IProcessResultImpl; 28 import android.media.Image; 29 import android.media.Image.Plane; 30 import android.media.ImageReader; 31 import android.media.ImageWriter; 32 import android.os.Handler; 33 import android.os.HandlerThread; 34 import android.os.IBinder; 35 import android.os.RemoteException; 36 import android.util.Log; 37 import android.view.Surface; 38 39 import com.android.internal.camera.flags.Flags; 40 41 import java.nio.ByteBuffer; 42 import java.util.HashSet; 43 import java.util.Iterator; 44 import java.util.List; 45 import java.util.concurrent.ConcurrentLinkedQueue; 46 47 // Jpeg compress input YUV and queue back in the client target surface. 48 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 49 public final static String TAG = "CameraExtensionJpeg"; 50 private final static int JPEG_QUEUE_SIZE = 1; 51 private final static int JPEG_APP_SEGMENT_SIZE = 64 * 1024; 52 53 private final Handler mHandler; 54 private final HandlerThread mHandlerThread; 55 private final ICaptureProcessorImpl mProcessor; 56 57 private ImageReader mYuvReader = null; 58 private ImageReader mPostviewYuvReader = null; 59 private android.hardware.camera2.extension.Size mResolution = null; 60 private android.hardware.camera2.extension.Size mPostviewResolution = null; 61 private int mFormat = -1; 62 private int mPostviewFormat = -1; 63 private int mCaptureFormat = -1; 64 private Surface mOutputSurface = null; 65 private ImageWriter mOutputWriter = null; 66 private Surface mPostviewOutputSurface = null; 67 private ImageWriter mPostviewOutputWriter = null; 68 69 private static final class JpegParameters { 70 public HashSet<Long> mTimeStamps = new HashSet<>(); 71 public int mRotation = JPEG_DEFAULT_ROTATION; // CW multiple of 90 degrees 72 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 73 } 74 75 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 76 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)77 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 78 mProcessor = processor; 79 mHandlerThread = new HandlerThread(TAG); 80 mHandlerThread.start(); 81 mHandler = new Handler(mHandlerThread.getLooper()); 82 } 83 close()84 public void close() { 85 mHandlerThread.quitSafely(); 86 87 if (mOutputWriter != null) { 88 mOutputWriter.close(); 89 mOutputWriter = null; 90 } 91 92 if (mYuvReader != null) { 93 mYuvReader.close(); 94 mYuvReader = null; 95 } 96 } 97 getJpegParameters(List<CaptureBundle> captureBundles)98 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 99 JpegParameters ret = new JpegParameters(); 100 if (!captureBundles.isEmpty()) { 101 // The quality and orientation settings must be equal for requests in a burst 102 103 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 104 if (jpegQuality != null) { 105 ret.mQuality = jpegQuality; 106 } else { 107 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 108 } 109 110 Integer orientation = captureBundles.get(0).captureResult.get( 111 CaptureResult.JPEG_ORIENTATION); 112 if (orientation != null) { 113 // The jpeg encoder expects CCW rotation, convert from CW 114 ret.mRotation = (360 - (orientation % 360)) / 90; 115 } else { 116 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 117 } 118 119 for (CaptureBundle bundle : captureBundles) { 120 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 121 if (timeStamp != null) { 122 ret.mTimeStamps.add(timeStamp); 123 } else { 124 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 125 } 126 } 127 } 128 129 return ret; 130 } 131 132 /** 133 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 134 * <p> 135 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 136 * each channel of Y, Cb, Cr.<br> 137 * The Y plane is assumed to have the same width and height of the entire 138 * image.<br> 139 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 140 * have dimensions (floor(width / 2), floor(height / 2)).<br> 141 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 142 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 143 * from byteBuffer[x * pixel_stride + y * row_stride]. 144 * <p> 145 * The pre-compression transformation is applied as follows: 146 * <ol> 147 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 148 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 149 * (width, height) is a no-op.</li> 150 * <li>The rotation is applied counter-clockwise relative to the coordinate 151 * space of the image, so a CCW rotation will appear CW when the image is 152 * rendered in scanline order. Only rotations which are multiples of 153 * 90-degrees are suppored, so the parameter 'rot90' specifies which 154 * multiple of 90 to rotate the image.</li> 155 * </ol> 156 * 157 * @param width the width of the image to compress 158 * @param height the height of the image to compress 159 * @param yBuf the buffer containing the Y component of the image 160 * @param yPStride the stride between adjacent pixels in the same row in 161 * yBuf 162 * @param yRStride the stride between adjacent rows in yBuf 163 * @param cbBuf the buffer containing the Cb component of the image 164 * @param cbPStride the stride between adjacent pixels in the same row in 165 * cbBuf 166 * @param cbRStride the stride between adjacent rows in cbBuf 167 * @param crBuf the buffer containing the Cr component of the image 168 * @param crPStride the stride between adjacent pixels in the same row in 169 * crBuf 170 * @param crRStride the stride between adjacent rows in crBuf 171 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 172 * This must have enough capacity to store the result, or an 173 * error code will be returned. 174 * @param outBufCapacity the capacity of outBuf 175 * @param quality the jpeg-quality (1-100) to use 176 * @param cropLeft left-edge of the bounds of the image to crop to before 177 * rotation 178 * @param cropTop top-edge of the bounds of the image to crop to before 179 * rotation 180 * @param cropRight right-edge of the bounds of the image to crop to before 181 * rotation 182 * @param cropBottom bottom-edge of the bounds of the image to crop to 183 * before rotation 184 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 185 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)186 private static native int compressJpegFromYUV420pNative( 187 int width, int height, 188 ByteBuffer yBuf, int yPStride, int yRStride, 189 ByteBuffer cbBuf, int cbPStride, int cbRStride, 190 ByteBuffer crBuf, int crPStride, int crRStride, 191 ByteBuffer outBuf, int outBufCapacity, 192 int quality, 193 int cropLeft, int cropTop, int cropRight, int cropBottom, 194 int rot90); 195 196 @Override process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, boolean isPostviewRequested)197 public void process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, 198 boolean isPostviewRequested) 199 throws RemoteException { 200 JpegParameters jpegParams = getJpegParameters(captureBundle); 201 try { 202 mJpegParameters.add(jpegParams); 203 mProcessor.process(captureBundle, captureCallback, isPostviewRequested); 204 } catch (Exception e) { 205 mJpegParameters.remove(jpegParams); 206 throw e; 207 } 208 } 209 onOutputSurface(Surface surface, int format)210 public void onOutputSurface(Surface surface, int format) throws RemoteException { 211 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(surface); 212 mCaptureFormat = surfaceInfo.mFormat; 213 mOutputSurface = surface; 214 initializePipeline(); 215 } 216 onPostviewOutputSurface(Surface surface)217 public void onPostviewOutputSurface(Surface surface) throws RemoteException { 218 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo = 219 CameraExtensionUtils.querySurface(surface); 220 mPostviewFormat = postviewSurfaceInfo.mFormat; 221 mPostviewOutputSurface = surface; 222 initializePostviewPipeline(); 223 } 224 225 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size, android.hardware.camera2.extension.Size postviewSize)226 public void onResolutionUpdate(android.hardware.camera2.extension.Size size, 227 android.hardware.camera2.extension.Size postviewSize) 228 throws RemoteException { 229 mResolution = size; 230 mPostviewResolution = postviewSize; 231 initializePipeline(); 232 } 233 onImageFormatUpdate(int format)234 public void onImageFormatUpdate(int format) throws RemoteException { 235 mFormat = format; 236 initializePipeline(); 237 } 238 initializePipeline()239 private void initializePipeline() throws RemoteException { 240 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 241 (mYuvReader == null)) { 242 if (mCaptureFormat == ImageFormat.YUV_420_888) { 243 // For the case when postview is JPEG and capture is YUV 244 mProcessor.onOutputSurface(mOutputSurface, mCaptureFormat); 245 } else { 246 // Jpeg/blobs are expected to be configured with (w*h)x1.5 + 64k Jpeg APP1 segment 247 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 248 ImageFormat.JPEG, 249 (mResolution.width * mResolution.height * 3) / 2 250 + JPEG_APP_SEGMENT_SIZE, 1); 251 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, 252 mFormat, JPEG_QUEUE_SIZE); 253 mYuvReader.setOnImageAvailableListener( 254 new YuvCallback(mYuvReader, mOutputWriter), mHandler); 255 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 256 } 257 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 258 mProcessor.onImageFormatUpdate(ImageFormat.YUV_420_888); 259 } 260 } 261 initializePostviewPipeline()262 private void initializePostviewPipeline() throws RemoteException { 263 if ((mFormat != -1) && (mPostviewOutputSurface != null) && (mPostviewResolution != null) 264 && (mPostviewYuvReader == null)) { 265 if (mPostviewFormat == ImageFormat.YUV_420_888) { 266 // For the case when postview is YUV and capture is JPEG 267 mProcessor.onPostviewOutputSurface(mPostviewOutputSurface); 268 } else { 269 // Jpeg/blobs are expected to be configured with (w*h)x1 270 mPostviewOutputWriter = ImageWriter.newInstance(mPostviewOutputSurface, 271 1/*maxImages*/, ImageFormat.JPEG, 272 mPostviewResolution.width * mPostviewResolution.height, 1); 273 mPostviewYuvReader = ImageReader.newInstance(mPostviewResolution.width, 274 mPostviewResolution.height, mFormat, JPEG_QUEUE_SIZE); 275 mPostviewYuvReader.setOnImageAvailableListener( 276 new YuvCallback(mPostviewYuvReader, mPostviewOutputWriter), mHandler); 277 mProcessor.onPostviewOutputSurface(mPostviewYuvReader.getSurface()); 278 } 279 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 280 mProcessor.onImageFormatUpdate(ImageFormat.YUV_420_888); 281 } 282 } 283 284 @Override asBinder()285 public IBinder asBinder() { 286 throw new UnsupportedOperationException("Binder IPC not supported!"); 287 } 288 289 private class YuvCallback implements ImageReader.OnImageAvailableListener { 290 private ImageReader mImageReader; 291 private ImageWriter mImageWriter; 292 YuvCallback(ImageReader imageReader, ImageWriter imageWriter)293 public YuvCallback(ImageReader imageReader, ImageWriter imageWriter) { 294 mImageReader = imageReader; 295 mImageWriter = imageWriter; 296 } 297 298 @Override onImageAvailable(ImageReader reader)299 public void onImageAvailable(ImageReader reader) { 300 Image yuvImage = null; 301 Image jpegImage = null; 302 try { 303 yuvImage = mImageReader.acquireNextImage(); 304 jpegImage = mImageWriter.dequeueInputImage(); 305 } catch (IllegalStateException e) { 306 if (yuvImage != null) { 307 yuvImage.close(); 308 } 309 if (jpegImage != null) { 310 jpegImage.close(); 311 } 312 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 313 return; 314 } 315 316 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 317 jpegBuffer.clear(); 318 // Jpeg/blobs are expected to be configured with (w*h)x1 319 int jpegCapacity = jpegImage.getWidth(); 320 321 Plane lumaPlane = yuvImage.getPlanes()[0]; 322 Plane crPlane = yuvImage.getPlanes()[1]; 323 Plane cbPlane = yuvImage.getPlanes()[2]; 324 325 ConcurrentLinkedQueue<JpegParameters> jpegParameters = 326 new ConcurrentLinkedQueue(mJpegParameters); 327 Iterator<JpegParameters> jpegIter = jpegParameters.iterator(); 328 JpegParameters jpegParams = null; 329 while(jpegIter.hasNext()) { 330 JpegParameters currentParams = jpegIter.next(); 331 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 332 jpegParams = currentParams; 333 jpegIter.remove(); 334 break; 335 } 336 } 337 if (jpegParams == null) { 338 if (jpegParameters.isEmpty()) { 339 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 340 + " and quality!"); 341 jpegParams = new JpegParameters(); 342 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 343 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 344 } else { 345 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 346 + " processed input!"); 347 Log.w(TAG, "Using values from the top of the queue!"); 348 jpegParams = jpegParameters.poll(); 349 } 350 } 351 352 compressJpegFromYUV420pNative( 353 yuvImage.getWidth(), yuvImage.getHeight(), 354 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 355 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 356 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 357 jpegBuffer, jpegCapacity, jpegParams.mQuality, 358 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 359 jpegParams.mRotation); 360 jpegImage.setTimestamp(yuvImage.getTimestamp()); 361 yuvImage.close(); 362 363 try { 364 mImageWriter.queueInputImage(jpegImage); 365 } catch (IllegalStateException e) { 366 Log.e(TAG, "Failed to queue encoded result!"); 367 } finally { 368 jpegImage.close(); 369 } 370 } 371 } 372 } 373